pmap.c revision 1.24
1 /* $NetBSD: pmap.c,v 1.24 2001/09/29 09:39:12 chris Exp $ */ 2 3 /* 4 * Copyright (c) 2001 Richard Earnshaw 5 * Copyright (c) 2001 Christopher Gilbert 6 * All rights reserved. 7 * 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the company nor the name of the author may be used to 14 * endorse or promote products derived from this software without specific 15 * prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 18 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /*- 31 * Copyright (c) 1999 The NetBSD Foundation, Inc. 32 * All rights reserved. 33 * 34 * This code is derived from software contributed to The NetBSD Foundation 35 * by Charles M. Hannum. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. All advertising materials mentioning features or use of this software 46 * must display the following acknowledgement: 47 * This product includes software developed by the NetBSD 48 * Foundation, Inc. and its contributors. 49 * 4. Neither the name of The NetBSD Foundation nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 /* 67 * Copyright (c) 1994-1998 Mark Brinicombe. 68 * Copyright (c) 1994 Brini. 69 * All rights reserved. 70 * 71 * This code is derived from software written for Brini by Mark Brinicombe 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 3. All advertising materials mentioning features or use of this software 82 * must display the following acknowledgement: 83 * This product includes software developed by Mark Brinicombe. 84 * 4. The name of the author may not be used to endorse or promote products 85 * derived from this software without specific prior written permission. 86 * 87 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 88 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 89 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 90 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 91 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 92 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 93 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 94 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 95 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 96 * 97 * RiscBSD kernel project 98 * 99 * pmap.c 100 * 101 * Machine dependant vm stuff 102 * 103 * Created : 20/09/94 104 */ 105 106 /* 107 * Performance improvements, UVM changes, overhauls and part-rewrites 108 * were contributed by Neil A. Carson <neil (at) causality.com>. 109 */ 110 111 /* 112 * The dram block info is currently referenced from the bootconfig. 113 * This should be placed in a separate structure. 114 */ 115 116 /* 117 * Special compilation symbols 118 * PMAP_DEBUG - Build in pmap_debug_level code 119 */ 120 121 /* Include header files */ 122 123 #include "opt_pmap_debug.h" 124 #include "opt_ddb.h" 125 126 #include <sys/types.h> 127 #include <sys/param.h> 128 #include <sys/kernel.h> 129 #include <sys/systm.h> 130 #include <sys/proc.h> 131 #include <sys/malloc.h> 132 #include <sys/user.h> 133 #include <sys/pool.h> 134 #include <sys/cdefs.h> 135 136 #include <uvm/uvm.h> 137 138 #include <machine/bootconfig.h> 139 #include <machine/bus.h> 140 #include <machine/pmap.h> 141 #include <machine/pcb.h> 142 #include <machine/param.h> 143 #include <machine/katelib.h> 144 145 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.24 2001/09/29 09:39:12 chris Exp $"); 146 #ifdef PMAP_DEBUG 147 #define PDEBUG(_lev_,_stat_) \ 148 if (pmap_debug_level >= (_lev_)) \ 149 ((_stat_)) 150 int pmap_debug_level = -2; 151 152 /* 153 * for switching to potentially finer grained debugging 154 */ 155 #define PDB_FOLLOW 0x0001 156 #define PDB_INIT 0x0002 157 #define PDB_ENTER 0x0004 158 #define PDB_REMOVE 0x0008 159 #define PDB_CREATE 0x0010 160 #define PDB_PTPAGE 0x0020 161 #define PDB_ASN 0x0040 162 #define PDB_BITS 0x0080 163 #define PDB_COLLECT 0x0100 164 #define PDB_PROTECT 0x0200 165 #define PDB_BOOTSTRAP 0x1000 166 #define PDB_PARANOIA 0x2000 167 #define PDB_WIRING 0x4000 168 #define PDB_PVDUMP 0x8000 169 170 int debugmap = 0; 171 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 172 #define NPDEBUG(_lev_,_stat_) \ 173 if (pmapdebug & (_lev_)) \ 174 ((_stat_)) 175 176 #else /* PMAP_DEBUG */ 177 #define PDEBUG(_lev_,_stat_) /* Nothing */ 178 #define PDEBUG(_lev_,_stat_) /* Nothing */ 179 #endif /* PMAP_DEBUG */ 180 181 struct pmap kernel_pmap_store; 182 183 /* 184 * pool that pmap structures are allocated from 185 */ 186 187 struct pool pmap_pmap_pool; 188 189 pagehook_t page_hook0; 190 pagehook_t page_hook1; 191 char *memhook; 192 pt_entry_t msgbufpte; 193 extern caddr_t msgbufaddr; 194 195 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 196 /* 197 * locking data structures 198 */ 199 200 static struct lock pmap_main_lock; 201 static struct simplelock pvalloc_lock; 202 #ifdef LOCKDEBUG 203 #define PMAP_MAP_TO_HEAD_LOCK() \ 204 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 205 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 206 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 207 208 #define PMAP_HEAD_TO_MAP_LOCK() \ 209 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 210 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 211 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 212 #else 213 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 214 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 215 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 216 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 217 #endif /* LOCKDEBUG */ 218 219 /* 220 * pv_page management structures: locked by pvalloc_lock 221 */ 222 223 TAILQ_HEAD(pv_pagelist, pv_page); 224 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 225 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 226 static int pv_nfpvents; /* # of free pv entries */ 227 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 228 static vaddr_t pv_cachedva; /* cached VA for later use */ 229 230 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 231 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 232 /* high water mark */ 233 234 /* 235 * local prototypes 236 */ 237 238 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 239 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 240 #define ALLOCPV_NEED 0 /* need PV now */ 241 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 242 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 243 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 244 static void pmap_enter_pv __P((struct pv_head *, 245 struct pv_entry *, struct pmap *, 246 vaddr_t, struct vm_page *, int)); 247 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 248 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 249 static void pmap_free_pv_doit __P((struct pv_entry *)); 250 static void pmap_free_pvpage __P((void)); 251 static boolean_t pmap_is_curpmap __P((struct pmap *)); 252 static struct pv_entry *pmap_remove_pv __P((struct pv_head *, struct pmap *, 253 vaddr_t)); 254 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 255 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 256 257 vsize_t npages; 258 259 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 260 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 261 __inline static void pmap_clearbit __P((paddr_t, unsigned int)); 262 __inline static boolean_t pmap_testbit __P((paddr_t, unsigned int)); 263 264 extern paddr_t physical_start; 265 extern paddr_t physical_freestart; 266 extern paddr_t physical_end; 267 extern paddr_t physical_freeend; 268 extern unsigned int free_pages; 269 extern int max_processes; 270 271 vaddr_t virtual_start; 272 vaddr_t virtual_end; 273 274 vaddr_t avail_start; 275 vaddr_t avail_end; 276 277 extern pv_addr_t systempage; 278 279 #define ALLOC_PAGE_HOOK(x, s) \ 280 x.va = virtual_start; \ 281 x.pte = (pt_entry_t *)pmap_pte(pmap_kernel(), virtual_start); \ 282 virtual_start += s; 283 284 /* Variables used by the L1 page table queue code */ 285 SIMPLEQ_HEAD(l1pt_queue, l1pt); 286 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 287 int l1pt_static_queue_count; /* items in the static l1 queue */ 288 int l1pt_static_create_count; /* static l1 items created */ 289 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 290 int l1pt_queue_count; /* items in the l1 queue */ 291 int l1pt_create_count; /* stat - L1's create count */ 292 int l1pt_reuse_count; /* stat - L1's reused count */ 293 294 /* Local function prototypes (not used outside this file) */ 295 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 296 void map_pagetable __P((vaddr_t pagetable, vaddr_t va, 297 paddr_t pa, unsigned int flags)); 298 void pmap_copy_on_write __P((paddr_t pa)); 299 void pmap_pinit __P((struct pmap *)); 300 void pmap_freepagedir __P((struct pmap *)); 301 302 /* Other function prototypes */ 303 extern void bzero_page __P((vaddr_t)); 304 extern void bcopy_page __P((vaddr_t, vaddr_t)); 305 306 struct l1pt *pmap_alloc_l1pt __P((void)); 307 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 308 vaddr_t l2pa, boolean_t)); 309 310 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 311 static void pmap_unmap_ptes __P((struct pmap *)); 312 313 void pmap_vac_me_harder __P((struct pmap *, struct pv_head *, 314 pt_entry_t *, boolean_t)); 315 316 /* 317 * real definition of pv_entry. 318 */ 319 320 struct pv_entry { 321 struct pv_entry *pv_next; /* next pv_entry */ 322 struct pmap *pv_pmap; /* pmap where mapping lies */ 323 vaddr_t pv_va; /* virtual address for mapping */ 324 int pv_flags; /* flags */ 325 struct vm_page *pv_ptp; /* vm_page for the ptp */ 326 }; 327 328 /* 329 * pv_entrys are dynamically allocated in chunks from a single page. 330 * we keep track of how many pv_entrys are in use for each page and 331 * we can free pv_entry pages if needed. there is one lock for the 332 * entire allocation system. 333 */ 334 335 struct pv_page_info { 336 TAILQ_ENTRY(pv_page) pvpi_list; 337 struct pv_entry *pvpi_pvfree; 338 int pvpi_nfree; 339 }; 340 341 /* 342 * number of pv_entry's in a pv_page 343 * (note: won't work on systems where NPBG isn't a constant) 344 */ 345 346 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 347 sizeof(struct pv_entry)) 348 349 /* 350 * a pv_page: where pv_entrys are allocated from 351 */ 352 353 struct pv_page { 354 struct pv_page_info pvinfo; 355 struct pv_entry pvents[PVE_PER_PVPAGE]; 356 }; 357 358 #ifdef MYCROFT_HACK 359 int mycroft_hack = 0; 360 #endif 361 362 /* Function to set the debug level of the pmap code */ 363 364 #ifdef PMAP_DEBUG 365 void 366 pmap_debug(level) 367 int level; 368 { 369 pmap_debug_level = level; 370 printf("pmap_debug: level=%d\n", pmap_debug_level); 371 } 372 #endif /* PMAP_DEBUG */ 373 374 __inline static boolean_t 375 pmap_is_curpmap(struct pmap *pmap) 376 { 377 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 378 || (pmap == pmap_kernel())) 379 return (TRUE); 380 return (FALSE); 381 } 382 #include "isadma.h" 383 384 #if NISADMA > 0 385 /* 386 * Used to protect memory for ISA DMA bounce buffers. If, when loading 387 * pages into the system, memory intersects with any of these ranges, 388 * the intersecting memory will be loaded into a lower-priority free list. 389 */ 390 bus_dma_segment_t *pmap_isa_dma_ranges; 391 int pmap_isa_dma_nranges; 392 393 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 394 paddr_t *, psize_t *)); 395 396 /* 397 * Check if a memory range intersects with an ISA DMA range, and 398 * return the page-rounded intersection if it does. The intersection 399 * will be placed on a lower-priority free list. 400 */ 401 boolean_t 402 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 403 paddr_t pa; 404 psize_t size; 405 paddr_t *pap; 406 psize_t *sizep; 407 { 408 bus_dma_segment_t *ds; 409 int i; 410 411 if (pmap_isa_dma_ranges == NULL) 412 return (FALSE); 413 414 for (i = 0, ds = pmap_isa_dma_ranges; 415 i < pmap_isa_dma_nranges; i++, ds++) { 416 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 417 /* 418 * Beginning of region intersects with this range. 419 */ 420 *pap = trunc_page(pa); 421 *sizep = round_page(min(pa + size, 422 ds->ds_addr + ds->ds_len) - pa); 423 return (TRUE); 424 } 425 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 426 /* 427 * End of region intersects with this range. 428 */ 429 *pap = trunc_page(ds->ds_addr); 430 *sizep = round_page(min((pa + size) - ds->ds_addr, 431 ds->ds_len)); 432 return (TRUE); 433 } 434 } 435 436 /* 437 * No intersection found. 438 */ 439 return (FALSE); 440 } 441 #endif /* NISADMA > 0 */ 442 443 /* 444 * p v _ e n t r y f u n c t i o n s 445 */ 446 447 /* 448 * pv_entry allocation functions: 449 * the main pv_entry allocation functions are: 450 * pmap_alloc_pv: allocate a pv_entry structure 451 * pmap_free_pv: free one pv_entry 452 * pmap_free_pvs: free a list of pv_entrys 453 * 454 * the rest are helper functions 455 */ 456 457 /* 458 * pmap_alloc_pv: inline function to allocate a pv_entry structure 459 * => we lock pvalloc_lock 460 * => if we fail, we call out to pmap_alloc_pvpage 461 * => 3 modes: 462 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 463 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 464 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 465 * one now 466 * 467 * "try" is for optional functions like pmap_copy(). 468 */ 469 470 __inline static struct pv_entry * 471 pmap_alloc_pv(pmap, mode) 472 struct pmap *pmap; 473 int mode; 474 { 475 struct pv_page *pvpage; 476 struct pv_entry *pv; 477 478 simple_lock(&pvalloc_lock); 479 480 if (pv_freepages.tqh_first != NULL) { 481 pvpage = pv_freepages.tqh_first; 482 pvpage->pvinfo.pvpi_nfree--; 483 if (pvpage->pvinfo.pvpi_nfree == 0) { 484 /* nothing left in this one? */ 485 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 486 } 487 pv = pvpage->pvinfo.pvpi_pvfree; 488 #ifdef DIAGNOSTIC 489 if (pv == NULL) 490 panic("pmap_alloc_pv: pvpi_nfree off"); 491 #endif 492 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 493 pv_nfpvents--; /* took one from pool */ 494 } else { 495 pv = NULL; /* need more of them */ 496 } 497 498 /* 499 * if below low water mark or we didn't get a pv_entry we try and 500 * create more pv_entrys ... 501 */ 502 503 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 504 if (pv == NULL) 505 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 506 mode : ALLOCPV_NEED); 507 else 508 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 509 } 510 511 simple_unlock(&pvalloc_lock); 512 return(pv); 513 } 514 515 /* 516 * pmap_alloc_pvpage: maybe allocate a new pvpage 517 * 518 * if need_entry is false: try and allocate a new pv_page 519 * if need_entry is true: try and allocate a new pv_page and return a 520 * new pv_entry from it. if we are unable to allocate a pv_page 521 * we make a last ditch effort to steal a pv_page from some other 522 * mapping. if that fails, we panic... 523 * 524 * => we assume that the caller holds pvalloc_lock 525 */ 526 527 static struct pv_entry * 528 pmap_alloc_pvpage(pmap, mode) 529 struct pmap *pmap; 530 int mode; 531 { 532 struct vm_page *pg; 533 struct pv_page *pvpage; 534 struct pv_entry *pv; 535 int s; 536 537 /* 538 * if we need_entry and we've got unused pv_pages, allocate from there 539 */ 540 541 if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) { 542 543 /* move it to pv_freepages list */ 544 pvpage = pv_unusedpgs.tqh_first; 545 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 546 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 547 548 /* allocate a pv_entry */ 549 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 550 pv = pvpage->pvinfo.pvpi_pvfree; 551 #ifdef DIAGNOSTIC 552 if (pv == NULL) 553 panic("pmap_alloc_pvpage: pvpi_nfree off"); 554 #endif 555 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 556 557 pv_nfpvents--; /* took one from pool */ 558 return(pv); 559 } 560 561 /* 562 * see if we've got a cached unmapped VA that we can map a page in. 563 * if not, try to allocate one. 564 */ 565 566 567 if (pv_cachedva == 0) { 568 s = splvm(); 569 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 570 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 571 splx(s); 572 if (pv_cachedva == 0) { 573 return (NULL); 574 } 575 } 576 577 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 578 UVM_PGA_USERESERVE); 579 if (pg) 580 pg->flags &= ~PG_BUSY; /* never busy */ 581 582 if (pg == NULL) 583 return (NULL); 584 585 /* 586 * add a mapping for our new pv_page and free its entrys (save one!) 587 * 588 * NOTE: If we are allocating a PV page for the kernel pmap, the 589 * pmap is already locked! (...but entering the mapping is safe...) 590 */ 591 592 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL); 593 pmap_update(pmap_kernel()); 594 pvpage = (struct pv_page *) pv_cachedva; 595 pv_cachedva = 0; 596 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 597 } 598 599 /* 600 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 601 * 602 * => caller must hold pvalloc_lock 603 * => if need_entry is true, we allocate and return one pv_entry 604 */ 605 606 static struct pv_entry * 607 pmap_add_pvpage(pvp, need_entry) 608 struct pv_page *pvp; 609 boolean_t need_entry; 610 { 611 int tofree, lcv; 612 613 /* do we need to return one? */ 614 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 615 616 pvp->pvinfo.pvpi_pvfree = NULL; 617 pvp->pvinfo.pvpi_nfree = tofree; 618 for (lcv = 0 ; lcv < tofree ; lcv++) { 619 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 620 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 621 } 622 if (need_entry) 623 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 624 else 625 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 626 pv_nfpvents += tofree; 627 return((need_entry) ? &pvp->pvents[lcv] : NULL); 628 } 629 630 /* 631 * pmap_free_pv_doit: actually free a pv_entry 632 * 633 * => do not call this directly! instead use either 634 * 1. pmap_free_pv ==> free a single pv_entry 635 * 2. pmap_free_pvs => free a list of pv_entrys 636 * => we must be holding pvalloc_lock 637 */ 638 639 __inline static void 640 pmap_free_pv_doit(pv) 641 struct pv_entry *pv; 642 { 643 struct pv_page *pvp; 644 645 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 646 pv_nfpvents++; 647 pvp->pvinfo.pvpi_nfree++; 648 649 /* nfree == 1 => fully allocated page just became partly allocated */ 650 if (pvp->pvinfo.pvpi_nfree == 1) { 651 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 652 } 653 654 /* free it */ 655 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 656 pvp->pvinfo.pvpi_pvfree = pv; 657 658 /* 659 * are all pv_page's pv_entry's free? move it to unused queue. 660 */ 661 662 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 663 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 664 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 665 } 666 } 667 668 /* 669 * pmap_free_pv: free a single pv_entry 670 * 671 * => we gain the pvalloc_lock 672 */ 673 674 __inline static void 675 pmap_free_pv(pmap, pv) 676 struct pmap *pmap; 677 struct pv_entry *pv; 678 { 679 simple_lock(&pvalloc_lock); 680 pmap_free_pv_doit(pv); 681 682 /* 683 * Can't free the PV page if the PV entries were associated with 684 * the kernel pmap; the pmap is already locked. 685 */ 686 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 687 pmap != pmap_kernel()) 688 pmap_free_pvpage(); 689 690 simple_unlock(&pvalloc_lock); 691 } 692 693 /* 694 * pmap_free_pvs: free a list of pv_entrys 695 * 696 * => we gain the pvalloc_lock 697 */ 698 699 __inline static void 700 pmap_free_pvs(pmap, pvs) 701 struct pmap *pmap; 702 struct pv_entry *pvs; 703 { 704 struct pv_entry *nextpv; 705 706 simple_lock(&pvalloc_lock); 707 708 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 709 nextpv = pvs->pv_next; 710 pmap_free_pv_doit(pvs); 711 } 712 713 /* 714 * Can't free the PV page if the PV entries were associated with 715 * the kernel pmap; the pmap is already locked. 716 */ 717 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 718 pmap != pmap_kernel()) 719 pmap_free_pvpage(); 720 721 simple_unlock(&pvalloc_lock); 722 } 723 724 725 /* 726 * pmap_free_pvpage: try and free an unused pv_page structure 727 * 728 * => assume caller is holding the pvalloc_lock and that 729 * there is a page on the pv_unusedpgs list 730 * => if we can't get a lock on the kmem_map we try again later 731 * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows 732 * that if we can lock the kmem_map then we are not already 733 * holding kmem_object's lock. 734 */ 735 736 static void 737 pmap_free_pvpage() 738 { 739 int s; 740 struct vm_map *map; 741 struct vm_map_entry *dead_entries; 742 struct pv_page *pvp; 743 744 s = splvm(); /* protect kmem_map */ 745 746 pvp = pv_unusedpgs.tqh_first; 747 748 /* 749 * note: watch out for pv_initpage which is allocated out of 750 * kernel_map rather than kmem_map. 751 */ 752 if (pvp == pv_initpage) 753 map = kernel_map; 754 else 755 map = kmem_map; 756 757 if (vm_map_lock_try(map)) { 758 759 /* remove pvp from pv_unusedpgs */ 760 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 761 762 /* unmap the page */ 763 dead_entries = NULL; 764 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 765 &dead_entries); 766 vm_map_unlock(map); 767 768 if (dead_entries != NULL) 769 uvm_unmap_detach(dead_entries, 0); 770 771 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 772 } 773 774 if (pvp == pv_initpage) 775 /* no more initpage, we've freed it */ 776 pv_initpage = NULL; 777 778 splx(s); 779 } 780 781 /* 782 * main pv_entry manipulation functions: 783 * pmap_enter_pv: enter a mapping onto a pv_head list 784 * pmap_remove_pv: remove a mappiing from a pv_head list 785 * 786 * NOTE: pmap_enter_pv expects to lock the pvh itself 787 * pmap_remove_pv expects te caller to lock the pvh before calling 788 */ 789 790 /* 791 * pmap_enter_pv: enter a mapping onto a pv_head lst 792 * 793 * => caller should hold the proper lock on pmap_main_lock 794 * => caller should have pmap locked 795 * => we will gain the lock on the pv_head and allocate the new pv_entry 796 * => caller should adjust ptp's wire_count before calling 797 * => caller should not adjust pmap's wire_count 798 */ 799 800 __inline static void 801 pmap_enter_pv(pvh, pve, pmap, va, ptp, flags) 802 struct pv_head *pvh; 803 struct pv_entry *pve; /* preallocated pve for us to use */ 804 struct pmap *pmap; 805 vaddr_t va; 806 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 807 int flags; 808 { 809 pve->pv_pmap = pmap; 810 pve->pv_va = va; 811 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 812 pve->pv_flags = flags; 813 simple_lock(&pvh->pvh_lock); /* lock pv_head */ 814 pve->pv_next = pvh->pvh_list; /* add to ... */ 815 pvh->pvh_list = pve; /* ... locked list */ 816 simple_unlock(&pvh->pvh_lock); /* unlock, done! */ 817 if (pve->pv_flags & PT_W) 818 ++pmap->pm_stats.wired_count; 819 } 820 821 /* 822 * pmap_remove_pv: try to remove a mapping from a pv_list 823 * 824 * => caller should hold proper lock on pmap_main_lock 825 * => pmap should be locked 826 * => caller should hold lock on pv_head [so that attrs can be adjusted] 827 * => caller should adjust ptp's wire_count and free PTP if needed 828 * => caller should NOT adjust pmap's wire_count 829 * => we return the removed pve 830 */ 831 832 __inline static struct pv_entry * 833 pmap_remove_pv(pvh, pmap, va) 834 struct pv_head *pvh; 835 struct pmap *pmap; 836 vaddr_t va; 837 { 838 struct pv_entry *pve, **prevptr; 839 840 prevptr = &pvh->pvh_list; /* previous pv_entry pointer */ 841 pve = *prevptr; 842 while (pve) { 843 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 844 *prevptr = pve->pv_next; /* remove it! */ 845 if (pve->pv_flags & PT_W) 846 --pmap->pm_stats.wired_count; 847 break; 848 } 849 prevptr = &pve->pv_next; /* previous pointer */ 850 pve = pve->pv_next; /* advance */ 851 } 852 return(pve); /* return removed pve */ 853 } 854 855 /* 856 * 857 * pmap_modify_pv: Update pv flags 858 * 859 * => caller should hold lock on pv_head [so that attrs can be adjusted] 860 * => caller should NOT adjust pmap's wire_count 861 * => we return the old flags 862 * 863 * Modify a physical-virtual mapping in the pv table 864 */ 865 866 /*__inline */ u_int 867 pmap_modify_pv(pmap, va, pvh, bic_mask, eor_mask) 868 struct pmap *pmap; 869 vaddr_t va; 870 struct pv_head *pvh; 871 u_int bic_mask; 872 u_int eor_mask; 873 { 874 struct pv_entry *npv; 875 u_int flags, oflags; 876 877 /* 878 * There is at least one VA mapping this page. 879 */ 880 881 for (npv = pvh->pvh_list; npv; npv = npv->pv_next) { 882 if (pmap == npv->pv_pmap && va == npv->pv_va) { 883 oflags = npv->pv_flags; 884 npv->pv_flags = flags = 885 ((oflags & ~bic_mask) ^ eor_mask); 886 if ((flags ^ oflags) & PT_W) { 887 if (flags & PT_W) 888 ++pmap->pm_stats.wired_count; 889 else 890 --pmap->pm_stats.wired_count; 891 } 892 return (oflags); 893 } 894 } 895 return (0); 896 } 897 898 899 /* 900 * Map the specified level 2 pagetable into the level 1 page table for 901 * the given pmap to cover a chunk of virtual address space starting from the 902 * address specified. 903 */ 904 static /*__inline*/ void 905 pmap_map_in_l1(pmap, va, l2pa, selfref) 906 struct pmap *pmap; 907 vaddr_t va, l2pa; 908 boolean_t selfref; 909 { 910 vaddr_t ptva; 911 912 /* Calculate the index into the L1 page table. */ 913 ptva = (va >> PDSHIFT) & ~3; 914 915 PDEBUG(0, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 916 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 917 918 /* Map page table into the L1. */ 919 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 920 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 921 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 922 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 923 924 PDEBUG(0, printf("pt self reference %lx in %lx\n", 925 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 926 927 /* Map the page table into the page table area. */ 928 if (selfref) { 929 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 930 L2_PTE_NC_NB(l2pa, AP_KRW); 931 } 932 /* XXX should be a purge */ 933 /* cpu_tlb_flushD();*/ 934 } 935 936 #if 0 937 static /*__inline*/ void 938 pmap_unmap_in_l1(pmap, va) 939 struct pmap *pmap; 940 vaddr_t va; 941 { 942 vaddr_t ptva; 943 944 /* Calculate the index into the L1 page table. */ 945 ptva = (va >> PDSHIFT) & ~3; 946 947 /* Unmap page table from the L1. */ 948 pmap->pm_pdir[ptva + 0] = 0; 949 pmap->pm_pdir[ptva + 1] = 0; 950 pmap->pm_pdir[ptva + 2] = 0; 951 pmap->pm_pdir[ptva + 3] = 0; 952 953 /* Unmap the page table from the page table area. */ 954 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 955 956 /* XXX should be a purge */ 957 /* cpu_tlb_flushD();*/ 958 } 959 #endif 960 961 962 /* 963 * Used to map a range of physical addresses into kernel 964 * virtual address space. 965 * 966 * For now, VM is already on, we only need to map the 967 * specified memory. 968 */ 969 vaddr_t 970 pmap_map(va, spa, epa, prot) 971 vaddr_t va, spa, epa; 972 int prot; 973 { 974 while (spa < epa) { 975 pmap_kenter_pa(va, spa, prot); 976 va += NBPG; 977 spa += NBPG; 978 } 979 pmap_update(pmap_kernel()); 980 return(va); 981 } 982 983 984 /* 985 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 986 * 987 * bootstrap the pmap system. This is called from initarm and allows 988 * the pmap system to initailise any structures it requires. 989 * 990 * Currently this sets up the kernel_pmap that is statically allocated 991 * and also allocated virtual addresses for certain page hooks. 992 * Currently the only one page hook is allocated that is used 993 * to zero physical pages of memory. 994 * It also initialises the start and end address of the kernel data space. 995 */ 996 extern paddr_t physical_freestart; 997 extern paddr_t physical_freeend; 998 999 char *boot_head; 1000 1001 void 1002 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1003 pd_entry_t *kernel_l1pt; 1004 pv_addr_t kernel_ptpt; 1005 { 1006 int loop; 1007 paddr_t start, end; 1008 #if NISADMA > 0 1009 paddr_t istart; 1010 psize_t isize; 1011 #endif 1012 1013 pmap_kernel()->pm_pdir = kernel_l1pt; 1014 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1015 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1016 simple_lock_init(&pmap_kernel()->pm_lock); 1017 pmap_kernel()->pm_obj.pgops = NULL; 1018 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1019 pmap_kernel()->pm_obj.uo_npages = 0; 1020 pmap_kernel()->pm_obj.uo_refs = 1; 1021 1022 /* 1023 * Initialize PAGE_SIZE-dependent variables. 1024 */ 1025 uvm_setpagesize(); 1026 1027 npages = 0; 1028 loop = 0; 1029 while (loop < bootconfig.dramblocks) { 1030 start = (paddr_t)bootconfig.dram[loop].address; 1031 end = start + (bootconfig.dram[loop].pages * NBPG); 1032 if (start < physical_freestart) 1033 start = physical_freestart; 1034 if (end > physical_freeend) 1035 end = physical_freeend; 1036 #if 0 1037 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1038 #endif 1039 #if NISADMA > 0 1040 if (pmap_isa_dma_range_intersect(start, end - start, 1041 &istart, &isize)) { 1042 /* 1043 * Place the pages that intersect with the 1044 * ISA DMA range onto the ISA DMA free list. 1045 */ 1046 #if 0 1047 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1048 istart + isize - 1); 1049 #endif 1050 uvm_page_physload(atop(istart), 1051 atop(istart + isize), atop(istart), 1052 atop(istart + isize), VM_FREELIST_ISADMA); 1053 npages += atop(istart + isize) - atop(istart); 1054 1055 /* 1056 * Load the pieces that come before 1057 * the intersection into the default 1058 * free list. 1059 */ 1060 if (start < istart) { 1061 #if 0 1062 printf(" BEFORE 0x%lx -> 0x%lx\n", 1063 start, istart - 1); 1064 #endif 1065 uvm_page_physload(atop(start), 1066 atop(istart), atop(start), 1067 atop(istart), VM_FREELIST_DEFAULT); 1068 npages += atop(istart) - atop(start); 1069 } 1070 1071 /* 1072 * Load the pieces that come after 1073 * the intersection into the default 1074 * free list. 1075 */ 1076 if ((istart + isize) < end) { 1077 #if 0 1078 printf(" AFTER 0x%lx -> 0x%lx\n", 1079 (istart + isize), end - 1); 1080 #endif 1081 uvm_page_physload(atop(istart + isize), 1082 atop(end), atop(istart + isize), 1083 atop(end), VM_FREELIST_DEFAULT); 1084 npages += atop(end) - atop(istart + isize); 1085 } 1086 } else { 1087 uvm_page_physload(atop(start), atop(end), 1088 atop(start), atop(end), VM_FREELIST_DEFAULT); 1089 npages += atop(end) - atop(start); 1090 } 1091 #else /* NISADMA > 0 */ 1092 uvm_page_physload(atop(start), atop(end), 1093 atop(start), atop(end), VM_FREELIST_DEFAULT); 1094 npages += atop(end) - atop(start); 1095 #endif /* NISADMA > 0 */ 1096 ++loop; 1097 } 1098 1099 #ifdef MYCROFT_HACK 1100 printf("npages = %ld\n", npages); 1101 #endif 1102 1103 virtual_start = KERNEL_VM_BASE; 1104 virtual_end = virtual_start + KERNEL_VM_SIZE - 1; 1105 1106 ALLOC_PAGE_HOOK(page_hook0, NBPG); 1107 ALLOC_PAGE_HOOK(page_hook1, NBPG); 1108 1109 /* 1110 * The mem special device needs a virtual hook but we don't 1111 * need a pte 1112 */ 1113 memhook = (char *)virtual_start; 1114 virtual_start += NBPG; 1115 1116 msgbufaddr = (caddr_t)virtual_start; 1117 msgbufpte = (pt_entry_t)pmap_pte(pmap_kernel(), virtual_start); 1118 virtual_start += round_page(MSGBUFSIZE); 1119 1120 /* 1121 * init the static-global locks and global lists. 1122 */ 1123 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1124 simple_lock_init(&pvalloc_lock); 1125 TAILQ_INIT(&pv_freepages); 1126 TAILQ_INIT(&pv_unusedpgs); 1127 1128 /* 1129 * compute the number of pages we have and then allocate RAM 1130 * for each pages' pv_head and saved attributes. 1131 */ 1132 { 1133 int npages, lcv; 1134 vsize_t s; 1135 1136 npages = 0; 1137 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) 1138 npages += (vm_physmem[lcv].end - vm_physmem[lcv].start); 1139 s = (vsize_t) (sizeof(struct pv_head) * npages + 1140 sizeof(char) * npages); 1141 s = round_page(s); /* round up */ 1142 boot_head = (char *)uvm_pageboot_alloc(s); 1143 bzero((char *)boot_head, s); 1144 if (boot_head == 0) 1145 panic("pmap_init: unable to allocate pv_heads"); 1146 } 1147 1148 /* 1149 * initialize the pmap pool. 1150 */ 1151 1152 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1153 0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP); 1154 1155 cpu_cache_cleanD(); 1156 } 1157 1158 /* 1159 * void pmap_init(void) 1160 * 1161 * Initialize the pmap module. 1162 * Called by vm_init() in vm/vm_init.c in order to initialise 1163 * any structures that the pmap system needs to map virtual memory. 1164 */ 1165 1166 extern int physmem; 1167 1168 void 1169 pmap_init() 1170 { 1171 int lcv, i; 1172 1173 #ifdef MYCROFT_HACK 1174 printf("physmem = %d\n", physmem); 1175 #endif 1176 1177 /* 1178 * Set the available memory vars - These do not map to real memory 1179 * addresses and cannot as the physical memory is fragmented. 1180 * They are used by ps for %mem calculations. 1181 * One could argue whether this should be the entire memory or just 1182 * the memory that is useable in a user process. 1183 */ 1184 avail_start = 0; 1185 avail_end = physmem * NBPG; 1186 1187 /* allocate pv_head stuff first */ 1188 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1189 vm_physmem[lcv].pmseg.pvhead = (struct pv_head *)boot_head; 1190 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.pvhead + 1191 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1192 for (i = 0; 1193 i < (vm_physmem[lcv].end - vm_physmem[lcv].start); i++) { 1194 simple_lock_init( 1195 &vm_physmem[lcv].pmseg.pvhead[i].pvh_lock); 1196 } 1197 } 1198 1199 /* now allocate attrs */ 1200 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1201 vm_physmem[lcv].pmseg.attrs = (char *) boot_head; 1202 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.attrs + 1203 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1204 } 1205 1206 /* 1207 * now we need to free enough pv_entry structures to allow us to get 1208 * the kmem_map/kmem_object allocated and inited (done after this 1209 * function is finished). to do this we allocate one bootstrap page out 1210 * of kernel_map and use it to provide an initial pool of pv_entry 1211 * structures. we never free this page. 1212 */ 1213 1214 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1215 if (pv_initpage == NULL) 1216 panic("pmap_init: pv_initpage"); 1217 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1218 pv_nfpvents = 0; 1219 (void) pmap_add_pvpage(pv_initpage, FALSE); 1220 1221 #ifdef MYCROFT_HACK 1222 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1223 printf("physseg[%d] pvent=%p attrs=%p start=%ld end=%ld\n", 1224 lcv, 1225 vm_physmem[lcv].pmseg.pvent, vm_physmem[lcv].pmseg.attrs, 1226 vm_physmem[lcv].start, vm_physmem[lcv].end); 1227 } 1228 #endif 1229 pmap_initialized = TRUE; 1230 1231 /* Initialise our L1 page table queues and counters */ 1232 SIMPLEQ_INIT(&l1pt_static_queue); 1233 l1pt_static_queue_count = 0; 1234 l1pt_static_create_count = 0; 1235 SIMPLEQ_INIT(&l1pt_queue); 1236 l1pt_queue_count = 0; 1237 l1pt_create_count = 0; 1238 l1pt_reuse_count = 0; 1239 } 1240 1241 /* 1242 * pmap_postinit() 1243 * 1244 * This routine is called after the vm and kmem subsystems have been 1245 * initialised. This allows the pmap code to perform any initialisation 1246 * that can only be done one the memory allocation is in place. 1247 */ 1248 1249 void 1250 pmap_postinit() 1251 { 1252 int loop; 1253 struct l1pt *pt; 1254 1255 #ifdef PMAP_STATIC_L1S 1256 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1257 #else /* PMAP_STATIC_L1S */ 1258 for (loop = 0; loop < max_processes; ++loop) { 1259 #endif /* PMAP_STATIC_L1S */ 1260 /* Allocate a L1 page table */ 1261 pt = pmap_alloc_l1pt(); 1262 if (!pt) 1263 panic("Cannot allocate static L1 page tables\n"); 1264 1265 /* Clean it */ 1266 bzero((void *)pt->pt_va, PD_SIZE); 1267 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1268 /* Add the page table to the queue */ 1269 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1270 ++l1pt_static_queue_count; 1271 ++l1pt_static_create_count; 1272 } 1273 } 1274 1275 1276 /* 1277 * Create and return a physical map. 1278 * 1279 * If the size specified for the map is zero, the map is an actual physical 1280 * map, and may be referenced by the hardware. 1281 * 1282 * If the size specified is non-zero, the map will be used in software only, 1283 * and is bounded by that size. 1284 */ 1285 1286 pmap_t 1287 pmap_create() 1288 { 1289 struct pmap *pmap; 1290 1291 /* 1292 * Fetch pmap entry from the pool 1293 */ 1294 1295 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1296 /* XXX is this really needed! */ 1297 memset(pmap, 0, sizeof(*pmap)); 1298 1299 simple_lock_init(&pmap->pm_obj.vmobjlock); 1300 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1301 TAILQ_INIT(&pmap->pm_obj.memq); 1302 pmap->pm_obj.uo_npages = 0; 1303 pmap->pm_obj.uo_refs = 1; 1304 pmap->pm_stats.wired_count = 0; 1305 pmap->pm_stats.resident_count = 1; 1306 1307 /* Now init the machine part of the pmap */ 1308 pmap_pinit(pmap); 1309 return(pmap); 1310 } 1311 1312 /* 1313 * pmap_alloc_l1pt() 1314 * 1315 * This routine allocates physical and virtual memory for a L1 page table 1316 * and wires it. 1317 * A l1pt structure is returned to describe the allocated page table. 1318 * 1319 * This routine is allowed to fail if the required memory cannot be allocated. 1320 * In this case NULL is returned. 1321 */ 1322 1323 struct l1pt * 1324 pmap_alloc_l1pt(void) 1325 { 1326 paddr_t pa; 1327 vaddr_t va; 1328 struct l1pt *pt; 1329 int error; 1330 struct vm_page *m; 1331 pt_entry_t *ptes; 1332 1333 /* Allocate virtual address space for the L1 page table */ 1334 va = uvm_km_valloc(kernel_map, PD_SIZE); 1335 if (va == 0) { 1336 #ifdef DIAGNOSTIC 1337 printf("pmap: Cannot allocate pageable memory for L1\n"); 1338 #endif /* DIAGNOSTIC */ 1339 return(NULL); 1340 } 1341 1342 /* Allocate memory for the l1pt structure */ 1343 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1344 1345 /* 1346 * Allocate pages from the VM system. 1347 */ 1348 TAILQ_INIT(&pt->pt_plist); 1349 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1350 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1351 if (error) { 1352 #ifdef DIAGNOSTIC 1353 printf("pmap: Cannot allocate physical memory for L1 (%d)\n", 1354 error); 1355 #endif /* DIAGNOSTIC */ 1356 /* Release the resources we already have claimed */ 1357 free(pt, M_VMPMAP); 1358 uvm_km_free(kernel_map, va, PD_SIZE); 1359 return(NULL); 1360 } 1361 1362 /* Map our physical pages into our virtual space */ 1363 pt->pt_va = va; 1364 m = pt->pt_plist.tqh_first; 1365 ptes = pmap_map_ptes(pmap_kernel()); 1366 while (m && va < (pt->pt_va + PD_SIZE)) { 1367 pa = VM_PAGE_TO_PHYS(m); 1368 1369 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1370 1371 /* Revoke cacheability and bufferability */ 1372 /* XXX should be done better than this */ 1373 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1374 1375 va += NBPG; 1376 m = m->pageq.tqe_next; 1377 } 1378 pmap_unmap_ptes(pmap_kernel()); 1379 pmap_update(pmap_kernel()); 1380 1381 #ifdef DIAGNOSTIC 1382 if (m) 1383 panic("pmap_alloc_l1pt: pglist not empty\n"); 1384 #endif /* DIAGNOSTIC */ 1385 1386 pt->pt_flags = 0; 1387 return(pt); 1388 } 1389 1390 /* 1391 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1392 */ 1393 void 1394 pmap_free_l1pt(pt) 1395 struct l1pt *pt; 1396 { 1397 /* Separate the physical memory for the virtual space */ 1398 pmap_kremove(pt->pt_va, PD_SIZE); 1399 pmap_update(pmap_kernel()); 1400 1401 /* Return the physical memory */ 1402 uvm_pglistfree(&pt->pt_plist); 1403 1404 /* Free the virtual space */ 1405 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1406 1407 /* Free the l1pt structure */ 1408 free(pt, M_VMPMAP); 1409 } 1410 1411 /* 1412 * Allocate a page directory. 1413 * This routine will either allocate a new page directory from the pool 1414 * of L1 page tables currently held by the kernel or it will allocate 1415 * a new one via pmap_alloc_l1pt(). 1416 * It will then initialise the l1 page table for use. 1417 */ 1418 int 1419 pmap_allocpagedir(pmap) 1420 struct pmap *pmap; 1421 { 1422 paddr_t pa; 1423 struct l1pt *pt; 1424 pt_entry_t *pte; 1425 1426 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1427 1428 /* Do we have any spare L1's lying around ? */ 1429 if (l1pt_static_queue_count) { 1430 --l1pt_static_queue_count; 1431 pt = l1pt_static_queue.sqh_first; 1432 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1433 } else if (l1pt_queue_count) { 1434 --l1pt_queue_count; 1435 pt = l1pt_queue.sqh_first; 1436 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1437 ++l1pt_reuse_count; 1438 } else { 1439 pt = pmap_alloc_l1pt(); 1440 if (!pt) 1441 return(ENOMEM); 1442 ++l1pt_create_count; 1443 } 1444 1445 /* Store the pointer to the l1 descriptor in the pmap. */ 1446 pmap->pm_l1pt = pt; 1447 1448 /* Get the physical address of the start of the l1 */ 1449 pa = VM_PAGE_TO_PHYS(pt->pt_plist.tqh_first); 1450 1451 /* Store the virtual address of the l1 in the pmap. */ 1452 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1453 1454 /* Clean the L1 if it is dirty */ 1455 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1456 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1457 1458 /* Do we already have the kernel mappings ? */ 1459 if (!(pt->pt_flags & PTFLAG_KPT)) { 1460 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1461 1462 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1463 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1464 KERNEL_PD_SIZE); 1465 pt->pt_flags |= PTFLAG_KPT; 1466 } 1467 1468 /* Allocate a page table to map all the page tables for this pmap */ 1469 1470 #ifdef DIAGNOSTIC 1471 if (pmap->pm_vptpt) { 1472 /* XXX What if we have one already ? */ 1473 panic("pmap_allocpagedir: have pt already\n"); 1474 } 1475 #endif /* DIAGNOSTIC */ 1476 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1477 if (pmap->pm_vptpt == 0) { 1478 pmap_freepagedir(pmap); 1479 return(ENOMEM); 1480 } 1481 1482 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1483 pmap->pm_pptpt &= PG_FRAME; 1484 /* Revoke cacheability and bufferability */ 1485 /* XXX should be done better than this */ 1486 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1487 *pte = *pte & ~(PT_C | PT_B); 1488 1489 /* Wire in this page table */ 1490 pmap_map_in_l1(pmap, PROCESS_PAGE_TBLS_BASE, pmap->pm_pptpt, TRUE); 1491 1492 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1493 1494 /* 1495 * Map the kernel page tables for 0xf0000000 + 1496 * into the page table used to map the 1497 * pmap's page tables 1498 */ 1499 bcopy((char *)(PROCESS_PAGE_TBLS_BASE 1500 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) 1501 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1502 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1503 (KERNEL_PD_SIZE >> 2)); 1504 1505 return(0); 1506 } 1507 1508 1509 /* 1510 * Initialize a preallocated and zeroed pmap structure, 1511 * such as one in a vmspace structure. 1512 */ 1513 1514 void 1515 pmap_pinit(pmap) 1516 struct pmap *pmap; 1517 { 1518 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1519 1520 /* Keep looping until we succeed in allocating a page directory */ 1521 while (pmap_allocpagedir(pmap) != 0) { 1522 /* 1523 * Ok we failed to allocate a suitable block of memory for an 1524 * L1 page table. This means that either: 1525 * 1. 16KB of virtual address space could not be allocated 1526 * 2. 16KB of physically contiguous memory on a 16KB boundary 1527 * could not be allocated. 1528 * 1529 * Since we cannot fail we will sleep for a while and try 1530 * again. 1531 */ 1532 (void) ltsleep(&lbolt, PVM, "l1ptwait", hz >> 3, NULL); 1533 } 1534 1535 /* Map zero page for the pmap. This will also map the L2 for it */ 1536 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1537 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1538 pmap_update(pmap); 1539 } 1540 1541 1542 void 1543 pmap_freepagedir(pmap) 1544 struct pmap *pmap; 1545 { 1546 /* Free the memory used for the page table mapping */ 1547 if (pmap->pm_vptpt != 0) 1548 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1549 1550 /* junk the L1 page table */ 1551 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1552 /* Add the page table to the queue */ 1553 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1554 ++l1pt_static_queue_count; 1555 } else if (l1pt_queue_count < 8) { 1556 /* Add the page table to the queue */ 1557 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1558 ++l1pt_queue_count; 1559 } else 1560 pmap_free_l1pt(pmap->pm_l1pt); 1561 } 1562 1563 1564 /* 1565 * Retire the given physical map from service. 1566 * Should only be called if the map contains no valid mappings. 1567 */ 1568 1569 void 1570 pmap_destroy(pmap) 1571 struct pmap *pmap; 1572 { 1573 struct vm_page *page; 1574 int count; 1575 1576 if (pmap == NULL) 1577 return; 1578 1579 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1580 1581 /* 1582 * Drop reference count 1583 */ 1584 simple_lock(&pmap->pm_obj.vmobjlock); 1585 count = --pmap->pm_obj.uo_refs; 1586 simple_unlock(&pmap->pm_obj.vmobjlock); 1587 if (count > 0) { 1588 return; 1589 } 1590 1591 /* 1592 * reference count is zero, free pmap resources and then free pmap. 1593 */ 1594 1595 /* Remove the zero page mapping */ 1596 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1597 pmap_update(pmap); 1598 1599 /* 1600 * Free any page tables still mapped 1601 * This is only temporay until pmap_enter can count the number 1602 * of mappings made in a page table. Then pmap_remove() can 1603 * reduce the count and free the pagetable when the count 1604 * reaches zero. Note that entries in this list should match the 1605 * contents of the ptpt, however this is faster than walking a 1024 1606 * entries looking for pt's 1607 * taken from i386 pmap.c 1608 */ 1609 while (pmap->pm_obj.memq.tqh_first != NULL) { 1610 page = pmap->pm_obj.memq.tqh_first; 1611 #ifdef DIAGNOSTIC 1612 if (page->flags & PG_BUSY) 1613 panic("pmap_release: busy page table page"); 1614 #endif 1615 /* pmap_page_protect? currently no need for it. */ 1616 1617 page->wire_count = 0; 1618 uvm_pagefree(page); 1619 } 1620 1621 /* Free the page dir */ 1622 pmap_freepagedir(pmap); 1623 1624 /* return the pmap to the pool */ 1625 pool_put(&pmap_pmap_pool, pmap); 1626 } 1627 1628 1629 /* 1630 * void pmap_reference(struct pmap *pmap) 1631 * 1632 * Add a reference to the specified pmap. 1633 */ 1634 1635 void 1636 pmap_reference(pmap) 1637 struct pmap *pmap; 1638 { 1639 if (pmap == NULL) 1640 return; 1641 1642 simple_lock(&pmap->pm_lock); 1643 pmap->pm_obj.uo_refs++; 1644 simple_unlock(&pmap->pm_lock); 1645 } 1646 1647 /* 1648 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1649 * 1650 * Return the start and end addresses of the kernel's virtual space. 1651 * These values are setup in pmap_bootstrap and are updated as pages 1652 * are allocated. 1653 */ 1654 1655 void 1656 pmap_virtual_space(start, end) 1657 vaddr_t *start; 1658 vaddr_t *end; 1659 { 1660 *start = virtual_start; 1661 *end = virtual_end; 1662 } 1663 1664 1665 /* 1666 * Activate the address space for the specified process. If the process 1667 * is the current process, load the new MMU context. 1668 */ 1669 void 1670 pmap_activate(p) 1671 struct proc *p; 1672 { 1673 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1674 struct pcb *pcb = &p->p_addr->u_pcb; 1675 1676 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1677 (paddr_t *)&pcb->pcb_pagedir); 1678 1679 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1680 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1681 1682 if (p == curproc) { 1683 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1684 setttb((u_int)pcb->pcb_pagedir); 1685 } 1686 #if 0 1687 pmap->pm_pdchanged = FALSE; 1688 #endif 1689 } 1690 1691 1692 /* 1693 * Deactivate the address space of the specified process. 1694 */ 1695 void 1696 pmap_deactivate(p) 1697 struct proc *p; 1698 { 1699 } 1700 1701 1702 /* 1703 * pmap_clean_page() 1704 * 1705 * This is a local function used to work out the best strategy to clean 1706 * a single page referenced by its entry in the PV table. It's used by 1707 * pmap_copy_page, pmap_zero page and maybe some others later on. 1708 * 1709 * Its policy is effectively: 1710 * o If there are no mappings, we don't bother doing anything with the cache. 1711 * o If there is one mapping, we clean just that page. 1712 * o If there are multiple mappings, we clean the entire cache. 1713 * 1714 * So that some functions can be further optimised, it returns 0 if it didn't 1715 * clean the entire cache, or 1 if it did. 1716 * 1717 * XXX One bug in this routine is that if the pv_entry has a single page 1718 * mapped at 0x00000000 a whole cache clean will be performed rather than 1719 * just the 1 page. Since this should not occur in everyday use and if it does 1720 * it will just result in not the most efficient clean for the page. 1721 */ 1722 static int 1723 pmap_clean_page(pv, is_src) 1724 struct pv_entry *pv; 1725 boolean_t is_src; 1726 { 1727 struct pmap *pmap; 1728 struct pv_entry *npv; 1729 int cache_needs_cleaning = 0; 1730 vaddr_t page_to_clean = 0; 1731 1732 if (pv == NULL) 1733 /* nothing mapped in so nothing to flush */ 1734 return (0); 1735 1736 /* Since we flush the cache each time we change curproc, we 1737 * only need to flush the page if it is in the current pmap. 1738 */ 1739 if (curproc) 1740 pmap = curproc->p_vmspace->vm_map.pmap; 1741 else 1742 pmap = pmap_kernel(); 1743 1744 for (npv = pv; npv; npv = npv->pv_next) { 1745 if (npv->pv_pmap == pmap) { 1746 /* The page is mapped non-cacheable in 1747 * this map. No need to flush the cache. 1748 */ 1749 if (npv->pv_flags & PT_NC) { 1750 #ifdef DIAGNOSTIC 1751 if (cache_needs_cleaning) 1752 panic("pmap_clean_page: " 1753 "cache inconsistency"); 1754 #endif 1755 break; 1756 } 1757 #if 0 1758 /* This doesn't work, because pmap_protect 1759 doesn't flush changes on pages that it 1760 has write-protected. */ 1761 1762 /* If the page is not writeable and this 1763 is the source, then there is no need 1764 to flush it from the cache. */ 1765 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1766 continue; 1767 #endif 1768 if (cache_needs_cleaning){ 1769 page_to_clean = 0; 1770 break; 1771 } 1772 else 1773 page_to_clean = npv->pv_va; 1774 cache_needs_cleaning = 1; 1775 } 1776 } 1777 1778 if (page_to_clean) 1779 cpu_cache_purgeID_rng(page_to_clean, NBPG); 1780 else if (cache_needs_cleaning) { 1781 cpu_cache_purgeID(); 1782 return (1); 1783 } 1784 return (0); 1785 } 1786 1787 /* 1788 * pmap_find_pv() 1789 * 1790 * This is a local function that finds a PV head for a given physical page. 1791 * This is a common op, and this function removes loads of ifdefs in the code. 1792 */ 1793 static __inline struct pv_head * 1794 pmap_find_pvh(phys) 1795 paddr_t phys; 1796 { 1797 int bank, off; 1798 struct pv_head *pvh; 1799 1800 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) 1801 panic("pmap_find_pv: not a real page, phys=%lx\n", phys); 1802 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 1803 return (pvh); 1804 } 1805 1806 /* 1807 * pmap_zero_page() 1808 * 1809 * Zero a given physical page by mapping it at a page hook point. 1810 * In doing the zero page op, the page we zero is mapped cachable, as with 1811 * StrongARM accesses to non-cached pages are non-burst making writing 1812 * _any_ bulk data very slow. 1813 */ 1814 void 1815 pmap_zero_page(phys) 1816 paddr_t phys; 1817 { 1818 struct pv_head *pvh; 1819 1820 /* Get an entry for this page, and clean it it. */ 1821 pvh = pmap_find_pvh(phys); 1822 simple_lock(&pvh->pvh_lock); 1823 pmap_clean_page(pvh->pvh_list, FALSE); 1824 simple_unlock(&pvh->pvh_lock); 1825 1826 /* 1827 * Hook in the page, zero it, and purge the cache for that 1828 * zeroed page. Invalidate the TLB as needed. 1829 */ 1830 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1831 cpu_tlb_flushD_SE(page_hook0.va); 1832 bzero_page(page_hook0.va); 1833 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1834 } 1835 1836 /* pmap_pageidlezero() 1837 * 1838 * The same as above, except that we assume that the page is not 1839 * mapped. This means we never have to flush the cache first. Called 1840 * from the idle loop. 1841 */ 1842 boolean_t 1843 pmap_pageidlezero(phys) 1844 paddr_t phys; 1845 { 1846 int i, *ptr; 1847 boolean_t rv = TRUE; 1848 1849 #ifdef DIAGNOSTIC 1850 struct pv_head *pvh; 1851 1852 pvh = pmap_find_pvh(phys); 1853 if (pvh->pvh_list != NULL) 1854 panic("pmap_pageidlezero: zeroing mapped page\n"); 1855 #endif 1856 1857 /* 1858 * Hook in the page, zero it, and purge the cache for that 1859 * zeroed page. Invalidate the TLB as needed. 1860 */ 1861 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1862 cpu_tlb_flushD_SE(page_hook0.va); 1863 1864 for (i = 0, ptr = (int *)page_hook0.va; 1865 i < (NBPG / sizeof(int)); i++) { 1866 if (sched_whichqs != 0) { 1867 /* 1868 * A process has become ready. Abort now, 1869 * so we don't keep it waiting while we 1870 * do slow memory access to finish this 1871 * page. 1872 */ 1873 rv = FALSE; 1874 break; 1875 } 1876 *ptr++ = 0; 1877 } 1878 1879 if (rv) 1880 /* 1881 * if we aborted we'll rezero this page again later so don't 1882 * purge it unless we finished it 1883 */ 1884 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1885 return (rv); 1886 } 1887 1888 /* 1889 * pmap_copy_page() 1890 * 1891 * Copy one physical page into another, by mapping the pages into 1892 * hook points. The same comment regarding cachability as in 1893 * pmap_zero_page also applies here. 1894 */ 1895 void 1896 pmap_copy_page(src, dest) 1897 paddr_t src; 1898 paddr_t dest; 1899 { 1900 struct pv_head *src_pvh, *dest_pvh; 1901 boolean_t cleanedcache; 1902 1903 /* Get PV entries for the pages, and clean them if needed. */ 1904 src_pvh = pmap_find_pvh(src); 1905 1906 simple_lock(&src_pvh->pvh_lock); 1907 cleanedcache = pmap_clean_page(src_pvh->pvh_list, TRUE); 1908 simple_unlock(&src_pvh->pvh_lock); 1909 1910 if (cleanedcache == 0) { 1911 dest_pvh = pmap_find_pvh(dest); 1912 simple_lock(&dest_pvh->pvh_lock); 1913 pmap_clean_page(dest_pvh->pvh_list, FALSE); 1914 simple_unlock(&dest_pvh->pvh_lock); 1915 } 1916 /* 1917 * Map the pages into the page hook points, copy them, and purge 1918 * the cache for the appropriate page. Invalidate the TLB 1919 * as required. 1920 */ 1921 *page_hook0.pte = L2_PTE(src & PG_FRAME, AP_KRW); 1922 *page_hook1.pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1923 cpu_tlb_flushD_SE(page_hook0.va); 1924 cpu_tlb_flushD_SE(page_hook1.va); 1925 bcopy_page(page_hook0.va, page_hook1.va); 1926 cpu_cache_purgeD_rng(page_hook0.va, NBPG); 1927 cpu_cache_purgeD_rng(page_hook1.va, NBPG); 1928 } 1929 1930 /* 1931 * int pmap_next_phys_page(paddr_t *addr) 1932 * 1933 * Allocate another physical page returning true or false depending 1934 * on whether a page could be allocated. 1935 */ 1936 1937 paddr_t 1938 pmap_next_phys_page(addr) 1939 paddr_t addr; 1940 1941 { 1942 int loop; 1943 1944 if (addr < bootconfig.dram[0].address) 1945 return(bootconfig.dram[0].address); 1946 1947 loop = 0; 1948 1949 while (bootconfig.dram[loop].address != 0 1950 && addr > (bootconfig.dram[loop].address + bootconfig.dram[loop].pages * NBPG)) 1951 ++loop; 1952 1953 if (bootconfig.dram[loop].address == 0) 1954 return(0); 1955 1956 addr += NBPG; 1957 1958 if (addr >= (bootconfig.dram[loop].address + bootconfig.dram[loop].pages * NBPG)) { 1959 if (bootconfig.dram[loop + 1].address == 0) 1960 return(0); 1961 addr = bootconfig.dram[loop + 1].address; 1962 } 1963 1964 return(addr); 1965 } 1966 1967 #if 0 1968 void 1969 pmap_pte_addref(pmap, va) 1970 struct pmap *pmap; 1971 vaddr_t va; 1972 { 1973 pd_entry_t *pde; 1974 paddr_t pa; 1975 struct vm_page *m; 1976 1977 if (pmap == pmap_kernel()) 1978 return; 1979 1980 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1981 pa = pmap_pte_pa(pde); 1982 m = PHYS_TO_VM_PAGE(pa); 1983 ++m->wire_count; 1984 #ifdef MYCROFT_HACK 1985 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1986 pmap, va, pde, pa, m, m->wire_count); 1987 #endif 1988 } 1989 1990 void 1991 pmap_pte_delref(pmap, va) 1992 struct pmap *pmap; 1993 vaddr_t va; 1994 { 1995 pd_entry_t *pde; 1996 paddr_t pa; 1997 struct vm_page *m; 1998 1999 if (pmap == pmap_kernel()) 2000 return; 2001 2002 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 2003 pa = pmap_pte_pa(pde); 2004 m = PHYS_TO_VM_PAGE(pa); 2005 --m->wire_count; 2006 #ifdef MYCROFT_HACK 2007 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2008 pmap, va, pde, pa, m, m->wire_count); 2009 #endif 2010 if (m->wire_count == 0) { 2011 #ifdef MYCROFT_HACK 2012 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2013 pmap, va, pde, pa, m); 2014 #endif 2015 pmap_unmap_in_l1(pmap, va); 2016 uvm_pagefree(m); 2017 --pmap->pm_stats.resident_count; 2018 } 2019 } 2020 #else 2021 #define pmap_pte_addref(pmap, va) 2022 #define pmap_pte_delref(pmap, va) 2023 #endif 2024 2025 /* 2026 * Since we have a virtually indexed cache, we may need to inhibit caching if 2027 * there is more than one mapping and at least one of them is writable. 2028 * Since we purge the cache on every context switch, we only need to check for 2029 * other mappings within the same pmap, or kernel_pmap. 2030 * This function is also called when a page is unmapped, to possibly reenable 2031 * caching on any remaining mappings. 2032 * 2033 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2034 */ 2035 void 2036 pmap_vac_me_harder(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2037 boolean_t clear_cache) 2038 { 2039 struct pv_entry *pv, *npv; 2040 pt_entry_t *pte; 2041 int entries = 0; 2042 int writeable = 0; 2043 int cacheable_entries = 0; 2044 2045 pv = pvh->pvh_list; 2046 KASSERT(ptes != NULL); 2047 2048 /* 2049 * Count mappings and writable mappings in this pmap. 2050 * Keep a pointer to the first one. 2051 */ 2052 for (npv = pv; npv; npv = npv->pv_next) { 2053 /* Count mappings in the same pmap */ 2054 if (pmap == npv->pv_pmap) { 2055 if (entries++ == 0) 2056 pv = npv; 2057 /* Cacheable mappings */ 2058 if ((npv->pv_flags & PT_NC) == 0) 2059 cacheable_entries++; 2060 /* Writeable mappings */ 2061 if (npv->pv_flags & PT_Wr) 2062 ++writeable; 2063 } 2064 } 2065 2066 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2067 "writeable %d cacheable %d %s\n", pmap, entries, writeable, 2068 cacheable_entries, clear_cache ? "clean" : "no clean")); 2069 2070 /* 2071 * Enable or disable caching as necessary. 2072 * We do a quick check of the first PTE to avoid walking the list if 2073 * we're already in the right state. 2074 */ 2075 if (entries > 1 && writeable) { 2076 if (cacheable_entries == 0) 2077 return; 2078 if (pv->pv_flags & PT_NC) { 2079 #ifdef DIAGNOSTIC 2080 /* We have cacheable entries, but the first one 2081 isn't among them. Something is wrong. */ 2082 if (cacheable_entries) 2083 panic("pmap_vac_me_harder: " 2084 "cacheable inconsistent"); 2085 #endif 2086 return; 2087 } 2088 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2089 *pte &= ~(PT_C | PT_B); 2090 pv->pv_flags |= PT_NC; 2091 if (clear_cache && cacheable_entries < 4) { 2092 cpu_cache_purgeID_rng(pv->pv_va, NBPG); 2093 cpu_tlb_flushID_SE(pv->pv_va); 2094 } 2095 for (npv = pv->pv_next; npv; npv = npv->pv_next) { 2096 if (pmap == npv->pv_pmap && 2097 (npv->pv_flags & PT_NC) == 0) { 2098 ptes[arm_byte_to_page(npv->pv_va)] &= 2099 ~(PT_C | PT_B); 2100 npv->pv_flags |= PT_NC; 2101 if (clear_cache && cacheable_entries < 4) { 2102 cpu_cache_purgeID_rng(npv->pv_va, 2103 NBPG); 2104 cpu_tlb_flushID_SE(npv->pv_va); 2105 } 2106 } 2107 } 2108 if (clear_cache && cacheable_entries >= 4) { 2109 cpu_cache_purgeID(); 2110 cpu_tlb_flushID(); 2111 } 2112 } else if (entries > 0) { 2113 if ((pv->pv_flags & PT_NC) == 0) 2114 return; 2115 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2116 *pte |= (PT_C | PT_B); 2117 pv->pv_flags &= ~PT_NC; 2118 for (npv = pv->pv_next; npv; npv = npv->pv_next) { 2119 if (pmap == npv->pv_pmap && 2120 (npv->pv_flags & PT_NC)) { 2121 ptes[arm_byte_to_page(npv->pv_va)] |= 2122 (PT_C | PT_B); 2123 npv->pv_flags &= ~PT_NC; 2124 } 2125 } 2126 } 2127 } 2128 2129 /* 2130 * pmap_remove() 2131 * 2132 * pmap_remove is responsible for nuking a number of mappings for a range 2133 * of virtual address space in the current pmap. To do this efficiently 2134 * is interesting, because in a number of cases a wide virtual address 2135 * range may be supplied that contains few actual mappings. So, the 2136 * optimisations are: 2137 * 1. Try and skip over hunks of address space for which an L1 entry 2138 * does not exist. 2139 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2140 * maybe do just a partial cache clean. This path of execution is 2141 * complicated by the fact that the cache must be flushed _before_ 2142 * the PTE is nuked, being a VAC :-) 2143 * 3. Maybe later fast-case a single page, but I don't think this is 2144 * going to make _that_ much difference overall. 2145 */ 2146 2147 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2148 2149 void 2150 pmap_remove(pmap, sva, eva) 2151 struct pmap *pmap; 2152 vaddr_t sva; 2153 vaddr_t eva; 2154 { 2155 int cleanlist_idx = 0; 2156 struct pagelist { 2157 vaddr_t va; 2158 pt_entry_t *pte; 2159 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2160 pt_entry_t *pte = 0, *ptes; 2161 paddr_t pa; 2162 int pmap_active; 2163 struct pv_head *pvh; 2164 2165 /* Exit quick if there is no pmap */ 2166 if (!pmap) 2167 return; 2168 2169 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2170 2171 sva &= PG_FRAME; 2172 eva &= PG_FRAME; 2173 2174 /* 2175 * we lock in the pmap => pv_head direction 2176 */ 2177 PMAP_MAP_TO_HEAD_LOCK(); 2178 2179 ptes = pmap_map_ptes(pmap); 2180 /* Get a page table pointer */ 2181 while (sva < eva) { 2182 if (pmap_pde_v(pmap_pde(pmap, sva))) 2183 break; 2184 sva = (sva & PD_MASK) + NBPD; 2185 } 2186 2187 pte = &ptes[arm_byte_to_page(sva)]; 2188 /* Note if the pmap is active thus require cache and tlb cleans */ 2189 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2190 || (pmap == pmap_kernel())) 2191 pmap_active = 1; 2192 else 2193 pmap_active = 0; 2194 2195 /* Now loop along */ 2196 while (sva < eva) { 2197 /* Check if we can move to the next PDE (l1 chunk) */ 2198 if (!(sva & PT_MASK)) 2199 if (!pmap_pde_v(pmap_pde(pmap, sva))) { 2200 sva += NBPD; 2201 pte += arm_byte_to_page(NBPD); 2202 continue; 2203 } 2204 2205 /* We've found a valid PTE, so this page of PTEs has to go. */ 2206 if (pmap_pte_v(pte)) { 2207 int bank, off; 2208 2209 /* Update statistics */ 2210 --pmap->pm_stats.resident_count; 2211 2212 /* 2213 * Add this page to our cache remove list, if we can. 2214 * If, however the cache remove list is totally full, 2215 * then do a complete cache invalidation taking note 2216 * to backtrack the PTE table beforehand, and ignore 2217 * the lists in future because there's no longer any 2218 * point in bothering with them (we've paid the 2219 * penalty, so will carry on unhindered). Otherwise, 2220 * when we fall out, we just clean the list. 2221 */ 2222 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2223 pa = pmap_pte_pa(pte); 2224 2225 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2226 /* Add to the clean list. */ 2227 cleanlist[cleanlist_idx].pte = pte; 2228 cleanlist[cleanlist_idx].va = sva; 2229 cleanlist_idx++; 2230 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2231 int cnt; 2232 2233 /* Nuke everything if needed. */ 2234 if (pmap_active) { 2235 cpu_cache_purgeID(); 2236 cpu_tlb_flushID(); 2237 } 2238 2239 /* 2240 * Roll back the previous PTE list, 2241 * and zero out the current PTE. 2242 */ 2243 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2244 *cleanlist[cnt].pte = 0; 2245 pmap_pte_delref(pmap, cleanlist[cnt].va); 2246 } 2247 *pte = 0; 2248 pmap_pte_delref(pmap, sva); 2249 cleanlist_idx++; 2250 } else { 2251 /* 2252 * We've already nuked the cache and 2253 * TLB, so just carry on regardless, 2254 * and we won't need to do it again 2255 */ 2256 *pte = 0; 2257 pmap_pte_delref(pmap, sva); 2258 } 2259 2260 /* 2261 * Update flags. In a number of circumstances, 2262 * we could cluster a lot of these and do a 2263 * number of sequential pages in one go. 2264 */ 2265 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2266 struct pv_entry *pve; 2267 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2268 simple_lock(&pvh->pvh_lock); 2269 pve = pmap_remove_pv(pvh, pmap, sva); 2270 pmap_free_pv(pmap, pve); 2271 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2272 simple_unlock(&pvh->pvh_lock); 2273 } 2274 } 2275 sva += NBPG; 2276 pte++; 2277 } 2278 2279 pmap_unmap_ptes(pmap); 2280 /* 2281 * Now, if we've fallen through down to here, chances are that there 2282 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2283 */ 2284 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2285 u_int cnt; 2286 2287 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2288 if (pmap_active) { 2289 cpu_cache_purgeID_rng(cleanlist[cnt].va, NBPG); 2290 *cleanlist[cnt].pte = 0; 2291 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2292 } else 2293 *cleanlist[cnt].pte = 0; 2294 pmap_pte_delref(pmap, cleanlist[cnt].va); 2295 } 2296 } 2297 PMAP_MAP_TO_HEAD_UNLOCK(); 2298 } 2299 2300 /* 2301 * Routine: pmap_remove_all 2302 * Function: 2303 * Removes this physical page from 2304 * all physical maps in which it resides. 2305 * Reflects back modify bits to the pager. 2306 */ 2307 2308 void 2309 pmap_remove_all(pa) 2310 paddr_t pa; 2311 { 2312 struct pv_entry *pv, *npv; 2313 struct pv_head *pvh; 2314 struct pmap *pmap; 2315 pt_entry_t *pte, *ptes; 2316 2317 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", pa)); 2318 2319 /* set pv_head => pmap locking */ 2320 PMAP_HEAD_TO_MAP_LOCK(); 2321 2322 pvh = pmap_find_pvh(pa); 2323 simple_lock(&pvh->pvh_lock); 2324 2325 pv = pvh->pvh_list; 2326 if (pv == NULL) 2327 { 2328 PDEBUG(0, printf("free page\n")); 2329 simple_unlock(&pvh->pvh_lock); 2330 PMAP_HEAD_TO_MAP_UNLOCK(); 2331 return; 2332 } 2333 pmap_clean_page(pv, FALSE); 2334 2335 while (pv) { 2336 pmap = pv->pv_pmap; 2337 ptes = pmap_map_ptes(pmap); 2338 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2339 2340 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2341 pv->pv_va, pv->pv_flags)); 2342 #ifdef DEBUG 2343 if (!pmap_pde_v(pmap_pde(pmap, va)) || !pmap_pte_v(pte) 2344 || pmap_pte_pa(pte) != pa) 2345 panic("pmap_remove_all: bad mapping"); 2346 #endif /* DEBUG */ 2347 2348 /* 2349 * Update statistics 2350 */ 2351 --pmap->pm_stats.resident_count; 2352 2353 /* Wired bit */ 2354 if (pv->pv_flags & PT_W) 2355 --pmap->pm_stats.wired_count; 2356 2357 /* 2358 * Invalidate the PTEs. 2359 * XXX: should cluster them up and invalidate as many 2360 * as possible at once. 2361 */ 2362 2363 #ifdef needednotdone 2364 reduce wiring count on page table pages as references drop 2365 #endif 2366 2367 *pte = 0; 2368 pmap_pte_delref(pmap, pv->pv_va); 2369 2370 npv = pv->pv_next; 2371 pmap_free_pv(pmap, pv); 2372 pv = npv; 2373 pmap_unmap_ptes(pmap); 2374 } 2375 pvh->pvh_list = NULL; 2376 simple_unlock(&pvh->pvh_lock); 2377 PMAP_HEAD_TO_MAP_UNLOCK(); 2378 2379 PDEBUG(0, printf("done\n")); 2380 cpu_tlb_flushID(); 2381 } 2382 2383 2384 /* 2385 * Set the physical protection on the specified range of this map as requested. 2386 */ 2387 2388 void 2389 pmap_protect(pmap, sva, eva, prot) 2390 struct pmap *pmap; 2391 vaddr_t sva; 2392 vaddr_t eva; 2393 vm_prot_t prot; 2394 { 2395 pt_entry_t *pte = NULL, *ptes; 2396 int armprot; 2397 int flush = 0; 2398 paddr_t pa; 2399 int bank, off; 2400 struct pv_head *pvh; 2401 2402 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2403 pmap, sva, eva, prot)); 2404 2405 if (~prot & VM_PROT_READ) { 2406 /* Just remove the mappings. */ 2407 pmap_remove(pmap, sva, eva); 2408 return; 2409 } 2410 if (prot & VM_PROT_WRITE) { 2411 /* 2412 * If this is a read->write transition, just ignore it and let 2413 * uvm_fault() take care of it later. 2414 */ 2415 return; 2416 } 2417 2418 sva &= PG_FRAME; 2419 eva &= PG_FRAME; 2420 2421 /* Need to lock map->head */ 2422 PMAP_MAP_TO_HEAD_LOCK(); 2423 2424 ptes = pmap_map_ptes(pmap); 2425 /* 2426 * We need to acquire a pointer to a page table page before entering 2427 * the following loop. 2428 */ 2429 while (sva < eva) { 2430 if (pmap_pde_v(pmap_pde(pmap, sva))) 2431 break; 2432 sva = (sva & PD_MASK) + NBPD; 2433 } 2434 2435 pte = &ptes[arm_byte_to_page(sva)]; 2436 2437 while (sva < eva) { 2438 /* only check once in a while */ 2439 if ((sva & PT_MASK) == 0) { 2440 if (!pmap_pde_v(pmap_pde(pmap, sva))) { 2441 /* We can race ahead here, to the next pde. */ 2442 sva += NBPD; 2443 pte += arm_byte_to_page(NBPD); 2444 continue; 2445 } 2446 } 2447 2448 if (!pmap_pte_v(pte)) 2449 goto next; 2450 2451 flush = 1; 2452 2453 armprot = 0; 2454 if (sva < VM_MAXUSER_ADDRESS) 2455 armprot |= PT_AP(AP_U); 2456 else if (sva < VM_MAX_ADDRESS) 2457 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2458 *pte = (*pte & 0xfffff00f) | armprot; 2459 2460 pa = pmap_pte_pa(pte); 2461 2462 /* Get the physical page index */ 2463 2464 /* Clear write flag */ 2465 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2466 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2467 simple_lock(&pvh->pvh_lock); 2468 (void) pmap_modify_pv(pmap, sva, pvh, PT_Wr, 0); 2469 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2470 simple_unlock(&pvh->pvh_lock); 2471 } 2472 2473 next: 2474 sva += NBPG; 2475 pte++; 2476 } 2477 pmap_unmap_ptes(pmap); 2478 PMAP_MAP_TO_HEAD_UNLOCK(); 2479 if (flush) 2480 cpu_tlb_flushID(); 2481 } 2482 2483 /* 2484 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2485 * int flags) 2486 * 2487 * Insert the given physical page (p) at 2488 * the specified virtual address (v) in the 2489 * target physical map with the protection requested. 2490 * 2491 * If specified, the page will be wired down, meaning 2492 * that the related pte can not be reclaimed. 2493 * 2494 * NB: This is the only routine which MAY NOT lazy-evaluate 2495 * or lose information. That is, this routine must actually 2496 * insert this page into the given map NOW. 2497 */ 2498 2499 int 2500 pmap_enter(pmap, va, pa, prot, flags) 2501 struct pmap *pmap; 2502 vaddr_t va; 2503 paddr_t pa; 2504 vm_prot_t prot; 2505 int flags; 2506 { 2507 pt_entry_t *pte, *ptes; 2508 u_int npte; 2509 int bank, off; 2510 paddr_t opa; 2511 int nflags; 2512 boolean_t wired = (flags & PMAP_WIRED) != 0; 2513 struct pv_entry *pve; 2514 struct pv_head *pvh; 2515 int error; 2516 2517 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2518 va, pa, pmap, prot, wired)); 2519 2520 #ifdef DIAGNOSTIC 2521 /* Valid address ? */ 2522 if (va >= (KERNEL_VM_BASE + KERNEL_VM_SIZE)) 2523 panic("pmap_enter: too big"); 2524 if (pmap != pmap_kernel() && va != 0) { 2525 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2526 panic("pmap_enter: kernel page in user map"); 2527 } else { 2528 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2529 panic("pmap_enter: user page in kernel map"); 2530 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2531 panic("pmap_enter: entering PT page"); 2532 } 2533 #endif 2534 /* get lock */ 2535 PMAP_MAP_TO_HEAD_LOCK(); 2536 /* 2537 * Get a pointer to the pte for this virtual address. If the 2538 * pte pointer is NULL then we are missing the L2 page table 2539 * so we need to create one. 2540 */ 2541 /* XXX horrible hack to get us working with lockdebug */ 2542 simple_lock(&pmap->pm_obj.vmobjlock); 2543 pte = pmap_pte(pmap, va); 2544 if (!pte) { 2545 struct vm_page *ptp; 2546 2547 /* if failure is allowed then don't try too hard */ 2548 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2549 if (ptp == NULL) { 2550 if (flags & PMAP_CANFAIL) { 2551 error = ENOMEM; 2552 goto out; 2553 } 2554 panic("pmap_enter: get ptp failed"); 2555 } 2556 2557 pte = pmap_pte(pmap, va); 2558 #ifdef DIAGNOSTIC 2559 if (!pte) 2560 panic("pmap_enter: no pte"); 2561 #endif 2562 } 2563 2564 nflags = 0; 2565 if (prot & VM_PROT_WRITE) 2566 nflags |= PT_Wr; 2567 if (wired) 2568 nflags |= PT_W; 2569 2570 /* More debugging info */ 2571 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2572 *pte)); 2573 2574 /* Is the pte valid ? If so then this page is already mapped */ 2575 if (pmap_pte_v(pte)) { 2576 /* Get the physical address of the current page mapped */ 2577 opa = pmap_pte_pa(pte); 2578 2579 #ifdef MYCROFT_HACK 2580 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2581 #endif 2582 2583 /* Are we mapping the same page ? */ 2584 if (opa == pa) { 2585 /* All we must be doing is changing the protection */ 2586 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2587 va, pa)); 2588 2589 /* Has the wiring changed ? */ 2590 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2591 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2592 simple_lock(&pvh->pvh_lock); 2593 (void) pmap_modify_pv(pmap, va, pvh, 2594 PT_Wr | PT_W, nflags); 2595 simple_unlock(&pvh->pvh_lock); 2596 } else { 2597 pvh = NULL; 2598 } 2599 } else { 2600 /* We are replacing the page with a new one. */ 2601 cpu_cache_purgeID_rng(va, NBPG); 2602 2603 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2604 va, pa, opa)); 2605 2606 /* 2607 * If it is part of our managed memory then we 2608 * must remove it from the PV list 2609 */ 2610 if ((bank = vm_physseg_find(atop(opa), &off)) != -1) { 2611 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2612 simple_lock(&pvh->pvh_lock); 2613 pve = pmap_remove_pv(pvh, pmap, va); 2614 simple_unlock(&pvh->pvh_lock); 2615 } else { 2616 pve = NULL; 2617 } 2618 2619 goto enter; 2620 } 2621 } else { 2622 opa = 0; 2623 pve = NULL; 2624 pmap_pte_addref(pmap, va); 2625 2626 /* pte is not valid so we must be hooking in a new page */ 2627 ++pmap->pm_stats.resident_count; 2628 2629 enter: 2630 /* 2631 * Enter on the PV list if part of our managed memory 2632 */ 2633 bank = vm_physseg_find(atop(pa), &off); 2634 2635 if (pmap_initialized && (bank != -1)) { 2636 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2637 if (pve == NULL) { 2638 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2639 if (pve == NULL) { 2640 if (flags & PMAP_CANFAIL) { 2641 error = ENOMEM; 2642 goto out; 2643 } 2644 panic("pmap_enter: no pv entries available"); 2645 } 2646 } 2647 /* enter_pv locks pvh when adding */ 2648 pmap_enter_pv(pvh, pve, pmap, va, NULL, nflags); 2649 } else { 2650 pvh = NULL; 2651 if (pve != NULL) 2652 pmap_free_pv(pmap, pve); 2653 } 2654 } 2655 2656 #ifdef MYCROFT_HACK 2657 if (mycroft_hack) 2658 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2659 #endif 2660 2661 /* Construct the pte, giving the correct access. */ 2662 npte = (pa & PG_FRAME); 2663 2664 /* VA 0 is magic. */ 2665 if (pmap != pmap_kernel() && va != 0) 2666 npte |= PT_AP(AP_U); 2667 2668 if (pmap_initialized && bank != -1) { 2669 #ifdef DIAGNOSTIC 2670 if ((flags & VM_PROT_ALL) & ~prot) 2671 panic("pmap_enter: access_type exceeds prot"); 2672 #endif 2673 npte |= PT_C | PT_B; 2674 if (flags & VM_PROT_WRITE) { 2675 npte |= L2_SPAGE | PT_AP(AP_W); 2676 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 2677 } else if (flags & VM_PROT_ALL) { 2678 npte |= L2_SPAGE; 2679 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 2680 } else 2681 npte |= L2_INVAL; 2682 } else { 2683 if (prot & VM_PROT_WRITE) 2684 npte |= L2_SPAGE | PT_AP(AP_W); 2685 else if (prot & VM_PROT_ALL) 2686 npte |= L2_SPAGE; 2687 else 2688 npte |= L2_INVAL; 2689 } 2690 2691 #ifdef MYCROFT_HACK 2692 if (mycroft_hack) 2693 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2694 #endif 2695 2696 *pte = npte; 2697 2698 if (pmap_initialized && bank != -1) 2699 { 2700 boolean_t pmap_active = FALSE; 2701 /* XXX this will change once the whole of pmap_enter uses 2702 * map_ptes 2703 */ 2704 ptes = pmap_map_ptes(pmap); 2705 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2706 || (pmap == pmap_kernel())) 2707 pmap_active = TRUE; 2708 simple_lock(&pvh->pvh_lock); 2709 pmap_vac_me_harder(pmap, pvh, ptes, pmap_active); 2710 simple_unlock(&pvh->pvh_lock); 2711 pmap_unmap_ptes(pmap); 2712 } 2713 2714 /* Better flush the TLB ... */ 2715 cpu_tlb_flushID_SE(va); 2716 error = 0; 2717 out: 2718 simple_unlock(&pmap->pm_obj.vmobjlock); 2719 PMAP_MAP_TO_HEAD_UNLOCK(); 2720 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2721 2722 return error; 2723 } 2724 2725 void 2726 pmap_kenter_pa(va, pa, prot) 2727 vaddr_t va; 2728 paddr_t pa; 2729 vm_prot_t prot; 2730 { 2731 struct pmap *pmap = pmap_kernel(); 2732 pt_entry_t *pte; 2733 struct vm_page *pg; 2734 2735 if (!pmap_pde_v(pmap_pde(pmap, va))) { 2736 2737 /* 2738 * For the kernel pmaps it would be better to ensure 2739 * that they are always present, and to grow the 2740 * kernel as required. 2741 */ 2742 2743 /* must lock the pmap */ 2744 simple_lock(&(pmap_kernel()->pm_obj.vmobjlock)); 2745 /* Allocate a page table */ 2746 pg = uvm_pagealloc(&(pmap_kernel()->pm_obj), 0, NULL, 2747 UVM_PGA_USERESERVE | UVM_PGA_ZERO); 2748 if (pg == NULL) { 2749 panic("pmap_kenter_pa: no free pages"); 2750 } 2751 pg->flags &= ~PG_BUSY; /* never busy */ 2752 2753 /* Wire this page table into the L1. */ 2754 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(pg), TRUE); 2755 simple_unlock(&(pmap_kernel()->pm_obj.vmobjlock)); 2756 } 2757 pte = vtopte(va); 2758 KASSERT(!pmap_pte_v(pte)); 2759 *pte = L2_PTE(pa, AP_KRW); 2760 } 2761 2762 void 2763 pmap_kremove(va, len) 2764 vaddr_t va; 2765 vsize_t len; 2766 { 2767 pt_entry_t *pte; 2768 2769 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2770 2771 /* 2772 * We assume that we will only be called with small 2773 * regions of memory. 2774 */ 2775 2776 KASSERT(pmap_pde_v(pmap_pde(pmap_kernel(), va))); 2777 pte = vtopte(va); 2778 cpu_cache_purgeID_rng(va, PAGE_SIZE); 2779 *pte = 0; 2780 cpu_tlb_flushID_SE(va); 2781 } 2782 } 2783 2784 /* 2785 * pmap_page_protect: 2786 * 2787 * Lower the permission for all mappings to a given page. 2788 */ 2789 2790 void 2791 pmap_page_protect(pg, prot) 2792 struct vm_page *pg; 2793 vm_prot_t prot; 2794 { 2795 paddr_t pa = VM_PAGE_TO_PHYS(pg); 2796 2797 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", pa, prot)); 2798 2799 switch(prot) { 2800 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2801 case VM_PROT_READ|VM_PROT_WRITE: 2802 return; 2803 2804 case VM_PROT_READ: 2805 case VM_PROT_READ|VM_PROT_EXECUTE: 2806 pmap_copy_on_write(pa); 2807 break; 2808 2809 default: 2810 pmap_remove_all(pa); 2811 break; 2812 } 2813 } 2814 2815 2816 /* 2817 * Routine: pmap_unwire 2818 * Function: Clear the wired attribute for a map/virtual-address 2819 * pair. 2820 * In/out conditions: 2821 * The mapping must already exist in the pmap. 2822 */ 2823 2824 void 2825 pmap_unwire(pmap, va) 2826 struct pmap *pmap; 2827 vaddr_t va; 2828 { 2829 pt_entry_t *pte; 2830 paddr_t pa; 2831 int bank, off; 2832 struct pv_head *pvh; 2833 2834 /* 2835 * Make sure pmap is valid. -dct 2836 */ 2837 if (pmap == NULL) 2838 return; 2839 2840 /* Get the pte */ 2841 pte = pmap_pte(pmap, va); 2842 if (!pte) 2843 return; 2844 2845 /* Extract the physical address of the page */ 2846 pa = pmap_pte_pa(pte); 2847 2848 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 2849 return; 2850 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2851 simple_lock(&pvh->pvh_lock); 2852 /* Update the wired bit in the pv entry for this page. */ 2853 (void) pmap_modify_pv(pmap, va, pvh, PT_W, 0); 2854 simple_unlock(&pvh->pvh_lock); 2855 } 2856 2857 /* 2858 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 2859 * 2860 * Return the pointer to a page table entry corresponding to the supplied 2861 * virtual address. 2862 * 2863 * The page directory is first checked to make sure that a page table 2864 * for the address in question exists and if it does a pointer to the 2865 * entry is returned. 2866 * 2867 * The way this works is that that the kernel page tables are mapped 2868 * into the memory map at ALT_PAGE_TBLS_BASE to ALT_PAGE_TBLS_BASE+4MB. 2869 * This allows page tables to be located quickly. 2870 */ 2871 pt_entry_t * 2872 pmap_pte(pmap, va) 2873 struct pmap *pmap; 2874 vaddr_t va; 2875 { 2876 pt_entry_t *ptp; 2877 pt_entry_t *result; 2878 2879 /* The pmap must be valid */ 2880 if (!pmap) 2881 return(NULL); 2882 2883 /* Return the address of the pte */ 2884 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 2885 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 2886 2887 /* Do we have a valid pde ? If not we don't have a page table */ 2888 if (!pmap_pde_v(pmap_pde(pmap, va))) { 2889 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 2890 pmap_pde(pmap, va))); 2891 return(NULL); 2892 } 2893 2894 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 2895 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2896 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2897 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME))); 2898 2899 /* 2900 * If the pmap is the kernel pmap or the pmap is the active one 2901 * then we can just return a pointer to entry relative to 2902 * PROCESS_PAGE_TBLS_BASE. 2903 * Otherwise we need to map the page tables to an alternative 2904 * address and reference them there. 2905 */ 2906 if (pmap == pmap_kernel() || pmap->pm_pptpt 2907 == (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2908 + ((PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) & 2909 ~3) + (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 2910 ptp = (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 2911 } else { 2912 struct proc *p = curproc; 2913 2914 /* If we don't have a valid curproc use proc0 */ 2915 /* Perhaps we should just use kernel_pmap instead */ 2916 if (p == NULL) 2917 p = &proc0; 2918 #ifdef DIAGNOSTIC 2919 /* 2920 * The pmap should always be valid for the process so 2921 * panic if it is not. 2922 */ 2923 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 2924 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 2925 va, p, p->p_vmspace); 2926 console_debugger(); 2927 } 2928 /* 2929 * The pmap for the current process should be mapped. If it 2930 * is not then we have a problem. 2931 */ 2932 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 2933 (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2934 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2935 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 2936 printf("pmap pagetable = P%08lx current = P%08x ", 2937 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2938 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2939 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & 2940 PG_FRAME)); 2941 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 2942 panic("pmap_pte: current and pmap mismatch\n"); 2943 } 2944 #endif 2945 2946 ptp = (pt_entry_t *)ALT_PAGE_TBLS_BASE; 2947 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 2948 pmap->pm_pptpt, FALSE); 2949 cpu_tlb_flushD(); 2950 } 2951 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 2952 ((va >> (PGSHIFT-2)) & ~3))); 2953 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 2954 return(result); 2955 } 2956 2957 /* 2958 * Routine: pmap_extract 2959 * Function: 2960 * Extract the physical page address associated 2961 * with the given map/virtual_address pair. 2962 */ 2963 boolean_t 2964 pmap_extract(pmap, va, pap) 2965 struct pmap *pmap; 2966 vaddr_t va; 2967 paddr_t *pap; 2968 { 2969 pt_entry_t *pte, *ptes; 2970 paddr_t pa; 2971 2972 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 2973 2974 /* 2975 * Get the pte for this virtual address. 2976 */ 2977 ptes = pmap_map_ptes(pmap); 2978 pte = &ptes[arm_byte_to_page(va)]; 2979 2980 /* 2981 * If there is no pte then there is no page table etc. 2982 * Is the pte valid ? If not then no paged is actually mapped here 2983 */ 2984 if (!pmap_pde_v(pmap_pde(pmap, va)) || !pmap_pte_v(pte)){ 2985 pmap_unmap_ptes(pmap); 2986 return (FALSE); 2987 } 2988 2989 /* Return the physical address depending on the PTE type */ 2990 /* XXX What about L1 section mappings ? */ 2991 if ((*(pte) & L2_MASK) == L2_LPAGE) { 2992 /* Extract the physical address from the pte */ 2993 pa = (*(pte)) & ~(L2_LPAGE_SIZE - 1); 2994 2995 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 2996 (pa | (va & (L2_LPAGE_SIZE - 1))))); 2997 2998 if (pap != NULL) 2999 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3000 } else { 3001 /* Extract the physical address from the pte */ 3002 pa = pmap_pte_pa(pte); 3003 3004 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3005 (pa | (va & ~PG_FRAME)))); 3006 3007 if (pap != NULL) 3008 *pap = pa | (va & ~PG_FRAME); 3009 } 3010 pmap_unmap_ptes(pmap); 3011 return (TRUE); 3012 } 3013 3014 3015 /* 3016 * Copy the range specified by src_addr/len from the source map to the 3017 * range dst_addr/len in the destination map. 3018 * 3019 * This routine is only advisory and need not do anything. 3020 */ 3021 3022 void 3023 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3024 struct pmap *dst_pmap; 3025 struct pmap *src_pmap; 3026 vaddr_t dst_addr; 3027 vsize_t len; 3028 vaddr_t src_addr; 3029 { 3030 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3031 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3032 } 3033 3034 #if defined(PMAP_DEBUG) 3035 void 3036 pmap_dump_pvlist(phys, m) 3037 vaddr_t phys; 3038 char *m; 3039 { 3040 struct pv_head *pvh; 3041 struct pv_entry *pv; 3042 int bank, off; 3043 3044 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) { 3045 printf("INVALID PA\n"); 3046 return; 3047 } 3048 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3049 simple_lock(&pvh->pvh_lock); 3050 printf("%s %08lx:", m, phys); 3051 if (pvh->pvh_list == NULL) { 3052 printf(" no mappings\n"); 3053 return; 3054 } 3055 3056 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) 3057 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3058 pv->pv_va, pv->pv_flags); 3059 3060 printf("\n"); 3061 simple_unlock(&pvh->pvh_lock); 3062 } 3063 3064 #endif /* PMAP_DEBUG */ 3065 3066 __inline static boolean_t 3067 pmap_testbit(pa, setbits) 3068 paddr_t pa; 3069 unsigned int setbits; 3070 { 3071 int bank, off; 3072 3073 PDEBUG(1, printf("pmap_testbit: pa=%08lx set=%08x\n", pa, setbits)); 3074 3075 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3076 return(FALSE); 3077 3078 /* 3079 * Check saved info only 3080 */ 3081 if (vm_physmem[bank].pmseg.attrs[off] & setbits) { 3082 PDEBUG(0, printf("pmap_attributes = %02x\n", 3083 vm_physmem[bank].pmseg.attrs[off])); 3084 return(TRUE); 3085 } 3086 3087 return(FALSE); 3088 } 3089 3090 static pt_entry_t * 3091 pmap_map_ptes(struct pmap *pmap) 3092 { 3093 struct proc *p; 3094 3095 /* the kernel's pmap is always accessible */ 3096 if (pmap == pmap_kernel()) { 3097 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE ; 3098 } 3099 3100 if (pmap_is_curpmap(pmap)) { 3101 simple_lock(&pmap->pm_obj.vmobjlock); 3102 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3103 } 3104 3105 p = curproc; 3106 3107 if (p == NULL) 3108 p = &proc0; 3109 3110 /* need to lock both curpmap and pmap: use ordered locking */ 3111 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3112 simple_lock(&pmap->pm_obj.vmobjlock); 3113 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3114 } else { 3115 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3116 simple_lock(&pmap->pm_obj.vmobjlock); 3117 } 3118 3119 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3120 pmap->pm_pptpt, FALSE); 3121 cpu_tlb_flushD(); 3122 return (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3123 } 3124 3125 /* 3126 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3127 */ 3128 3129 static void 3130 pmap_unmap_ptes(pmap) 3131 struct pmap *pmap; 3132 { 3133 if (pmap == pmap_kernel()) { 3134 return; 3135 } 3136 if (pmap_is_curpmap(pmap)) { 3137 simple_unlock(&pmap->pm_obj.vmobjlock); 3138 } else { 3139 simple_unlock(&pmap->pm_obj.vmobjlock); 3140 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3141 } 3142 } 3143 3144 /* 3145 * Modify pte bits for all ptes corresponding to the given physical address. 3146 * We use `maskbits' rather than `clearbits' because we're always passing 3147 * constants and the latter would require an extra inversion at run-time. 3148 */ 3149 3150 static void 3151 pmap_clearbit(pa, maskbits) 3152 paddr_t pa; 3153 unsigned int maskbits; 3154 { 3155 struct pv_entry *pv; 3156 struct pv_head *pvh; 3157 pt_entry_t *pte; 3158 vaddr_t va; 3159 int bank, off, tlbentry; 3160 3161 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3162 pa, maskbits)); 3163 3164 tlbentry = 0; 3165 3166 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3167 return; 3168 PMAP_HEAD_TO_MAP_LOCK(); 3169 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3170 simple_lock(&pvh->pvh_lock); 3171 3172 /* 3173 * Clear saved attributes (modify, reference) 3174 */ 3175 vm_physmem[bank].pmseg.attrs[off] &= ~maskbits; 3176 3177 if (pvh->pvh_list == NULL) { 3178 simple_unlock(&pvh->pvh_lock); 3179 PMAP_HEAD_TO_MAP_UNLOCK(); 3180 return; 3181 } 3182 3183 /* 3184 * Loop over all current mappings setting/clearing as appropos 3185 */ 3186 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) { 3187 va = pv->pv_va; 3188 pv->pv_flags &= ~maskbits; 3189 pte = pmap_pte(pv->pv_pmap, va); 3190 KASSERT(pte != NULL); 3191 if (maskbits & (PT_Wr|PT_M)) 3192 { 3193 if ((pv->pv_flags & PT_NC)) 3194 { 3195 /* 3196 * entry is not cacheable, so reenable the cache, 3197 * nothing to flush 3198 */ 3199 *pte |= (PT_C | PT_B); 3200 pv->pv_flags &= ~PT_NC; 3201 } else { 3202 /* 3203 * entry is cacheable check if pmap is current if it 3204 * is flush it, otherwise it won't be in the cache 3205 */ 3206 if (pmap_is_curpmap(pv->pv_pmap)) 3207 { 3208 /* entry is in current pmap purge it */ 3209 cpu_cache_purgeID_rng(pv->pv_va, NBPG); 3210 } 3211 } 3212 3213 /* make the pte read only */ 3214 *pte &= ~PT_AP(AP_W); 3215 3216 if (pmap_is_curpmap(pv->pv_pmap)) 3217 /* 3218 * if we had cacheable pte's we'd clean the pte out to 3219 * memory here 3220 */ 3221 /* 3222 * flush tlb entry as it's in the current pmap 3223 */ 3224 cpu_tlb_flushID_SE(pv->pv_va); 3225 3226 } 3227 if (maskbits & PT_H) 3228 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3229 } 3230 simple_unlock(&pvh->pvh_lock); 3231 PMAP_HEAD_TO_MAP_UNLOCK(); 3232 } 3233 3234 3235 boolean_t 3236 pmap_clear_modify(pg) 3237 struct vm_page *pg; 3238 { 3239 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3240 boolean_t rv; 3241 3242 PDEBUG(0, printf("pmap_clear_modify pa=%08lx\n", pa)); 3243 rv = pmap_testbit(pa, PT_M); 3244 pmap_clearbit(pa, PT_M); 3245 return rv; 3246 } 3247 3248 3249 boolean_t 3250 pmap_clear_reference(pg) 3251 struct vm_page *pg; 3252 { 3253 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3254 boolean_t rv; 3255 3256 PDEBUG(0, printf("pmap_clear_reference pa=%08lx\n", pa)); 3257 rv = pmap_testbit(pa, PT_H); 3258 pmap_clearbit(pa, PT_H); 3259 return rv; 3260 } 3261 3262 3263 void 3264 pmap_copy_on_write(pa) 3265 paddr_t pa; 3266 { 3267 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", pa)); 3268 pmap_clearbit(pa, PT_Wr); 3269 } 3270 3271 3272 boolean_t 3273 pmap_is_modified(pg) 3274 struct vm_page *pg; 3275 { 3276 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3277 boolean_t result; 3278 3279 result = pmap_testbit(pa, PT_M); 3280 PDEBUG(1, printf("pmap_is_modified pa=%08lx %x\n", pa, result)); 3281 return (result); 3282 } 3283 3284 3285 boolean_t 3286 pmap_is_referenced(pg) 3287 struct vm_page *pg; 3288 { 3289 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3290 boolean_t result; 3291 3292 result = pmap_testbit(pa, PT_H); 3293 PDEBUG(0, printf("pmap_is_referenced pa=%08lx %x\n", pa, result)); 3294 return (result); 3295 } 3296 3297 3298 int 3299 pmap_modified_emulation(pmap, va) 3300 struct pmap *pmap; 3301 vaddr_t va; 3302 { 3303 pt_entry_t *pte; 3304 paddr_t pa; 3305 int bank, off; 3306 struct pv_head *pvh; 3307 u_int flags; 3308 3309 PDEBUG(2, printf("pmap_modified_emulation\n")); 3310 3311 /* Get the pte */ 3312 pte = pmap_pte(pmap, va); 3313 if (!pte) { 3314 PDEBUG(2, printf("no pte\n")); 3315 return(0); 3316 } 3317 3318 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3319 3320 /* Check for a zero pte */ 3321 if (*pte == 0) 3322 return(0); 3323 3324 /* This can happen if user code tries to access kernel memory. */ 3325 if ((*pte & PT_AP(AP_W)) != 0) 3326 return (0); 3327 3328 /* Extract the physical address of the page */ 3329 pa = pmap_pte_pa(pte); 3330 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3331 return(0); 3332 3333 PMAP_HEAD_TO_MAP_LOCK(); 3334 /* Get the current flags for this page. */ 3335 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3336 /* XXX: needed if we hold head->map lock? */ 3337 simple_lock(&pvh->pvh_lock); 3338 3339 flags = pmap_modify_pv(pmap, va, pvh, 0, 0); 3340 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3341 3342 /* 3343 * Do the flags say this page is writable ? If not then it is a 3344 * genuine write fault. If yes then the write fault is our fault 3345 * as we did not reflect the write access in the PTE. Now we know 3346 * a write has occurred we can correct this and also set the 3347 * modified bit 3348 */ 3349 if (~flags & PT_Wr) { 3350 simple_unlock(&pvh->pvh_lock); 3351 PMAP_HEAD_TO_MAP_UNLOCK(); 3352 return(0); 3353 } 3354 3355 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3356 va, pte, *pte)); 3357 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 3358 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3359 PDEBUG(0, printf("->(%08x)\n", *pte)); 3360 3361 simple_unlock(&pvh->pvh_lock); 3362 PMAP_HEAD_TO_MAP_UNLOCK(); 3363 /* Return, indicating the problem has been dealt with */ 3364 cpu_tlb_flushID_SE(va); 3365 return(1); 3366 } 3367 3368 3369 int 3370 pmap_handled_emulation(pmap, va) 3371 struct pmap *pmap; 3372 vaddr_t va; 3373 { 3374 pt_entry_t *pte; 3375 paddr_t pa; 3376 int bank, off; 3377 3378 PDEBUG(2, printf("pmap_handled_emulation\n")); 3379 3380 /* Get the pte */ 3381 pte = pmap_pte(pmap, va); 3382 if (!pte) { 3383 PDEBUG(2, printf("no pte\n")); 3384 return(0); 3385 } 3386 3387 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3388 3389 /* Check for a zero pte */ 3390 if (*pte == 0) 3391 return(0); 3392 3393 /* This can happen if user code tries to access kernel memory. */ 3394 if ((*pte & L2_MASK) != L2_INVAL) 3395 return (0); 3396 3397 /* Extract the physical address of the page */ 3398 pa = pmap_pte_pa(pte); 3399 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3400 return(0); 3401 3402 /* 3403 * Ok we just enable the pte and mark the attibs as handled 3404 */ 3405 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3406 va, pte, *pte)); 3407 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 3408 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3409 PDEBUG(0, printf("->(%08x)\n", *pte)); 3410 3411 /* Return, indicating the problem has been dealt with */ 3412 cpu_tlb_flushID_SE(va); 3413 return(1); 3414 } 3415 3416 3417 3418 3419 /* 3420 * pmap_collect: free resources held by a pmap 3421 * 3422 * => optional function. 3423 * => called when a process is swapped out to free memory. 3424 */ 3425 3426 void 3427 pmap_collect(pmap) 3428 struct pmap *pmap; 3429 { 3430 } 3431 3432 /* 3433 * Routine: pmap_procwr 3434 * 3435 * Function: 3436 * Synchronize caches corresponding to [addr, addr+len) in p. 3437 * 3438 */ 3439 void 3440 pmap_procwr(p, va, len) 3441 struct proc *p; 3442 vaddr_t va; 3443 int len; 3444 { 3445 /* We only need to do anything if it is the current process. */ 3446 if (p == curproc) 3447 cpu_cache_syncI_rng(va, len); 3448 } 3449 /* 3450 * PTP functions 3451 */ 3452 3453 /* 3454 * pmap_steal_ptp: Steal a PTP from somewhere else. 3455 * 3456 * This is just a placeholder, for now we never steal. 3457 */ 3458 3459 static struct vm_page * 3460 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3461 { 3462 return (NULL); 3463 } 3464 3465 /* 3466 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3467 * 3468 * => pmap should NOT be pmap_kernel() 3469 * => pmap should be locked 3470 */ 3471 3472 static struct vm_page * 3473 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3474 { 3475 struct vm_page *ptp; 3476 3477 if (pmap_pde_v(pmap_pde(pmap, va))) { 3478 3479 /* valid... check hint (saves us a PA->PG lookup) */ 3480 #if 0 3481 if (pmap->pm_ptphint && 3482 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3483 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3484 return (pmap->pm_ptphint); 3485 #endif 3486 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3487 #ifdef DIAGNOSTIC 3488 if (ptp == NULL) 3489 panic("pmap_get_ptp: unmanaged user PTP"); 3490 #endif 3491 // pmap->pm_ptphint = ptp; 3492 return(ptp); 3493 } 3494 3495 /* allocate a new PTP (updates ptphint) */ 3496 return(pmap_alloc_ptp(pmap, va, just_try)); 3497 } 3498 3499 /* 3500 * pmap_alloc_ptp: allocate a PTP for a PMAP 3501 * 3502 * => pmap should already be locked by caller 3503 * => we use the ptp's wire_count to count the number of active mappings 3504 * in the PTP (we start it at one to prevent any chance this PTP 3505 * will ever leak onto the active/inactive queues) 3506 */ 3507 3508 /*__inline */ static struct vm_page * 3509 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3510 { 3511 struct vm_page *ptp; 3512 3513 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3514 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3515 if (ptp == NULL) { 3516 if (just_try) 3517 return (NULL); 3518 3519 ptp = pmap_steal_ptp(pmap, va); 3520 3521 if (ptp == NULL) 3522 return (NULL); 3523 /* Stole a page, zero it. */ 3524 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3525 } 3526 3527 /* got one! */ 3528 ptp->flags &= ~PG_BUSY; /* never busy */ 3529 ptp->wire_count = 1; /* no mappings yet */ 3530 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3531 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3532 // pmap->pm_ptphint = ptp; 3533 return (ptp); 3534 } 3535 3536 /* End of pmap.c */ 3537
Indexes created Fri Sep 26 08:10:20 GMT 2025