pmap.c revision 1.14.2.5
1 /* $NetBSD: pmap.c,v 1.14.2.5 2002/02/11 20:07:18 jdolecek Exp $ */ 2 3 /* 4 * Copyright (c) 2001 Richard Earnshaw 5 * Copyright (c) 2001 Christopher Gilbert 6 * All rights reserved. 7 * 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. The name of the company nor the name of the author may be used to 14 * endorse or promote products derived from this software without specific 15 * prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 18 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /*- 31 * Copyright (c) 1999 The NetBSD Foundation, Inc. 32 * All rights reserved. 33 * 34 * This code is derived from software contributed to The NetBSD Foundation 35 * by Charles M. Hannum. 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 1. Redistributions of source code must retain the above copyright 41 * notice, this list of conditions and the following disclaimer. 42 * 2. Redistributions in binary form must reproduce the above copyright 43 * notice, this list of conditions and the following disclaimer in the 44 * documentation and/or other materials provided with the distribution. 45 * 3. All advertising materials mentioning features or use of this software 46 * must display the following acknowledgement: 47 * This product includes software developed by the NetBSD 48 * Foundation, Inc. and its contributors. 49 * 4. Neither the name of The NetBSD Foundation nor the names of its 50 * contributors may be used to endorse or promote products derived 51 * from this software without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 54 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 55 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 56 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 57 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 58 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 59 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 60 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 61 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 62 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 63 * POSSIBILITY OF SUCH DAMAGE. 64 */ 65 66 /* 67 * Copyright (c) 1994-1998 Mark Brinicombe. 68 * Copyright (c) 1994 Brini. 69 * All rights reserved. 70 * 71 * This code is derived from software written for Brini by Mark Brinicombe 72 * 73 * Redistribution and use in source and binary forms, with or without 74 * modification, are permitted provided that the following conditions 75 * are met: 76 * 1. Redistributions of source code must retain the above copyright 77 * notice, this list of conditions and the following disclaimer. 78 * 2. Redistributions in binary form must reproduce the above copyright 79 * notice, this list of conditions and the following disclaimer in the 80 * documentation and/or other materials provided with the distribution. 81 * 3. All advertising materials mentioning features or use of this software 82 * must display the following acknowledgement: 83 * This product includes software developed by Mark Brinicombe. 84 * 4. The name of the author may not be used to endorse or promote products 85 * derived from this software without specific prior written permission. 86 * 87 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 88 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 89 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 90 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 91 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 92 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 93 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 94 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 95 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 96 * 97 * RiscBSD kernel project 98 * 99 * pmap.c 100 * 101 * Machine dependant vm stuff 102 * 103 * Created : 20/09/94 104 */ 105 106 /* 107 * Performance improvements, UVM changes, overhauls and part-rewrites 108 * were contributed by Neil A. Carson <neil (at) causality.com>. 109 */ 110 111 /* 112 * The dram block info is currently referenced from the bootconfig. 113 * This should be placed in a separate structure. 114 */ 115 116 /* 117 * Special compilation symbols 118 * PMAP_DEBUG - Build in pmap_debug_level code 119 */ 120 121 /* Include header files */ 122 123 #include "opt_pmap_debug.h" 124 #include "opt_ddb.h" 125 126 #include <sys/types.h> 127 #include <sys/param.h> 128 #include <sys/kernel.h> 129 #include <sys/systm.h> 130 #include <sys/proc.h> 131 #include <sys/malloc.h> 132 #include <sys/user.h> 133 #include <sys/pool.h> 134 #include <sys/cdefs.h> 135 136 #include <uvm/uvm.h> 137 138 #include <machine/bootconfig.h> 139 #include <machine/bus.h> 140 #include <machine/pmap.h> 141 #include <machine/pcb.h> 142 #include <machine/param.h> 143 #include <arm/arm32/katelib.h> 144 145 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.14.2.5 2002/02/11 20:07:18 jdolecek Exp $"); 146 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 153 /* 154 * for switching to potentially finer grained debugging 155 */ 156 #define PDB_FOLLOW 0x0001 157 #define PDB_INIT 0x0002 158 #define PDB_ENTER 0x0004 159 #define PDB_REMOVE 0x0008 160 #define PDB_CREATE 0x0010 161 #define PDB_PTPAGE 0x0020 162 #define PDB_ASN 0x0040 163 #define PDB_BITS 0x0080 164 #define PDB_COLLECT 0x0100 165 #define PDB_PROTECT 0x0200 166 #define PDB_BOOTSTRAP 0x1000 167 #define PDB_PARANOIA 0x2000 168 #define PDB_WIRING 0x4000 169 #define PDB_PVDUMP 0x8000 170 171 int debugmap = 0; 172 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 173 #define NPDEBUG(_lev_,_stat_) \ 174 if (pmapdebug & (_lev_)) \ 175 ((_stat_)) 176 177 #else /* PMAP_DEBUG */ 178 #define PDEBUG(_lev_,_stat_) /* Nothing */ 179 #define PDEBUG(_lev_,_stat_) /* Nothing */ 180 #endif /* PMAP_DEBUG */ 181 182 struct pmap kernel_pmap_store; 183 184 /* 185 * pool that pmap structures are allocated from 186 */ 187 188 struct pool pmap_pmap_pool; 189 190 pagehook_t page_hook0; 191 pagehook_t page_hook1; 192 char *memhook; 193 pt_entry_t msgbufpte; 194 extern caddr_t msgbufaddr; 195 196 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 197 /* 198 * locking data structures 199 */ 200 201 static struct lock pmap_main_lock; 202 static struct simplelock pvalloc_lock; 203 #ifdef LOCKDEBUG 204 #define PMAP_MAP_TO_HEAD_LOCK() \ 205 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 206 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 207 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 208 209 #define PMAP_HEAD_TO_MAP_LOCK() \ 210 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 211 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 212 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 213 #else 214 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 215 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 216 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 217 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 218 #endif /* LOCKDEBUG */ 219 220 /* 221 * pv_page management structures: locked by pvalloc_lock 222 */ 223 224 TAILQ_HEAD(pv_pagelist, pv_page); 225 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 226 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 227 static int pv_nfpvents; /* # of free pv entries */ 228 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 229 static vaddr_t pv_cachedva; /* cached VA for later use */ 230 231 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 232 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 233 /* high water mark */ 234 235 /* 236 * local prototypes 237 */ 238 239 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 240 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 241 #define ALLOCPV_NEED 0 /* need PV now */ 242 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 243 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 244 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 245 static void pmap_enter_pv __P((struct pv_head *, 246 struct pv_entry *, struct pmap *, 247 vaddr_t, struct vm_page *, int)); 248 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 249 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 250 static void pmap_free_pv_doit __P((struct pv_entry *)); 251 static void pmap_free_pvpage __P((void)); 252 static boolean_t pmap_is_curpmap __P((struct pmap *)); 253 static struct pv_entry *pmap_remove_pv __P((struct pv_head *, struct pmap *, 254 vaddr_t)); 255 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 256 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 257 258 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct pv_head *, 259 u_int, u_int)); 260 261 static void pmap_free_l1pt __P((struct l1pt *)); 262 static int pmap_allocpagedir __P((struct pmap *)); 263 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 264 static struct pv_head *pmap_find_pvh __P((paddr_t)); 265 static void pmap_remove_all __P((paddr_t)); 266 267 268 vsize_t npages; 269 270 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 271 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 272 __inline static void pmap_clearbit __P((paddr_t, unsigned int)); 273 __inline static boolean_t pmap_testbit __P((paddr_t, unsigned int)); 274 275 extern paddr_t physical_start; 276 extern paddr_t physical_freestart; 277 extern paddr_t physical_end; 278 extern paddr_t physical_freeend; 279 extern unsigned int free_pages; 280 extern int max_processes; 281 282 vaddr_t virtual_start; 283 vaddr_t virtual_end; 284 285 vaddr_t avail_start; 286 vaddr_t avail_end; 287 288 extern pv_addr_t systempage; 289 290 #define ALLOC_PAGE_HOOK(x, s) \ 291 x.va = virtual_start; \ 292 x.pte = (pt_entry_t *)pmap_pte(pmap_kernel(), virtual_start); \ 293 virtual_start += s; 294 295 /* Variables used by the L1 page table queue code */ 296 SIMPLEQ_HEAD(l1pt_queue, l1pt); 297 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 298 int l1pt_static_queue_count; /* items in the static l1 queue */ 299 int l1pt_static_create_count; /* static l1 items created */ 300 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 301 int l1pt_queue_count; /* items in the l1 queue */ 302 int l1pt_create_count; /* stat - L1's create count */ 303 int l1pt_reuse_count; /* stat - L1's reused count */ 304 305 /* Local function prototypes (not used outside this file) */ 306 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 307 void pmap_copy_on_write __P((paddr_t pa)); 308 void pmap_pinit __P((struct pmap *)); 309 void pmap_freepagedir __P((struct pmap *)); 310 311 /* Other function prototypes */ 312 extern void bzero_page __P((vaddr_t)); 313 extern void bcopy_page __P((vaddr_t, vaddr_t)); 314 315 struct l1pt *pmap_alloc_l1pt __P((void)); 316 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 317 vaddr_t l2pa, boolean_t)); 318 319 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 320 static void pmap_unmap_ptes __P((struct pmap *)); 321 322 __inline static void pmap_vac_me_harder __P((struct pmap *, struct pv_head *, 323 pt_entry_t *, boolean_t)); 324 static void pmap_vac_me_kpmap __P((struct pmap *, struct pv_head *, 325 pt_entry_t *, boolean_t)); 326 static void pmap_vac_me_user __P((struct pmap *, struct pv_head *, 327 pt_entry_t *, boolean_t)); 328 329 /* 330 * Cache enable bits in PTE to use on pages that are cacheable. 331 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 332 * can chose between write-through and write-back cacheing. 333 */ 334 pt_entry_t pte_cache_mode = (PT_C | PT_B); 335 336 /* 337 * real definition of pv_entry. 338 */ 339 340 struct pv_entry { 341 struct pv_entry *pv_next; /* next pv_entry */ 342 struct pmap *pv_pmap; /* pmap where mapping lies */ 343 vaddr_t pv_va; /* virtual address for mapping */ 344 int pv_flags; /* flags */ 345 struct vm_page *pv_ptp; /* vm_page for the ptp */ 346 }; 347 348 /* 349 * pv_entrys are dynamically allocated in chunks from a single page. 350 * we keep track of how many pv_entrys are in use for each page and 351 * we can free pv_entry pages if needed. there is one lock for the 352 * entire allocation system. 353 */ 354 355 struct pv_page_info { 356 TAILQ_ENTRY(pv_page) pvpi_list; 357 struct pv_entry *pvpi_pvfree; 358 int pvpi_nfree; 359 }; 360 361 /* 362 * number of pv_entry's in a pv_page 363 * (note: won't work on systems where NPBG isn't a constant) 364 */ 365 366 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 367 sizeof(struct pv_entry)) 368 369 /* 370 * a pv_page: where pv_entrys are allocated from 371 */ 372 373 struct pv_page { 374 struct pv_page_info pvinfo; 375 struct pv_entry pvents[PVE_PER_PVPAGE]; 376 }; 377 378 #ifdef MYCROFT_HACK 379 int mycroft_hack = 0; 380 #endif 381 382 /* Function to set the debug level of the pmap code */ 383 384 #ifdef PMAP_DEBUG 385 void 386 pmap_debug(level) 387 int level; 388 { 389 pmap_debug_level = level; 390 printf("pmap_debug: level=%d\n", pmap_debug_level); 391 } 392 #endif /* PMAP_DEBUG */ 393 394 __inline static boolean_t 395 pmap_is_curpmap(struct pmap *pmap) 396 { 397 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 398 || (pmap == pmap_kernel())) 399 return (TRUE); 400 return (FALSE); 401 } 402 #include "isadma.h" 403 404 #if NISADMA > 0 405 /* 406 * Used to protect memory for ISA DMA bounce buffers. If, when loading 407 * pages into the system, memory intersects with any of these ranges, 408 * the intersecting memory will be loaded into a lower-priority free list. 409 */ 410 bus_dma_segment_t *pmap_isa_dma_ranges; 411 int pmap_isa_dma_nranges; 412 413 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 414 paddr_t *, psize_t *)); 415 416 /* 417 * Check if a memory range intersects with an ISA DMA range, and 418 * return the page-rounded intersection if it does. The intersection 419 * will be placed on a lower-priority free list. 420 */ 421 boolean_t 422 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 423 paddr_t pa; 424 psize_t size; 425 paddr_t *pap; 426 psize_t *sizep; 427 { 428 bus_dma_segment_t *ds; 429 int i; 430 431 if (pmap_isa_dma_ranges == NULL) 432 return (FALSE); 433 434 for (i = 0, ds = pmap_isa_dma_ranges; 435 i < pmap_isa_dma_nranges; i++, ds++) { 436 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 437 /* 438 * Beginning of region intersects with this range. 439 */ 440 *pap = trunc_page(pa); 441 *sizep = round_page(min(pa + size, 442 ds->ds_addr + ds->ds_len) - pa); 443 return (TRUE); 444 } 445 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 446 /* 447 * End of region intersects with this range. 448 */ 449 *pap = trunc_page(ds->ds_addr); 450 *sizep = round_page(min((pa + size) - ds->ds_addr, 451 ds->ds_len)); 452 return (TRUE); 453 } 454 } 455 456 /* 457 * No intersection found. 458 */ 459 return (FALSE); 460 } 461 #endif /* NISADMA > 0 */ 462 463 /* 464 * p v _ e n t r y f u n c t i o n s 465 */ 466 467 /* 468 * pv_entry allocation functions: 469 * the main pv_entry allocation functions are: 470 * pmap_alloc_pv: allocate a pv_entry structure 471 * pmap_free_pv: free one pv_entry 472 * pmap_free_pvs: free a list of pv_entrys 473 * 474 * the rest are helper functions 475 */ 476 477 /* 478 * pmap_alloc_pv: inline function to allocate a pv_entry structure 479 * => we lock pvalloc_lock 480 * => if we fail, we call out to pmap_alloc_pvpage 481 * => 3 modes: 482 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 483 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 484 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 485 * one now 486 * 487 * "try" is for optional functions like pmap_copy(). 488 */ 489 490 __inline static struct pv_entry * 491 pmap_alloc_pv(pmap, mode) 492 struct pmap *pmap; 493 int mode; 494 { 495 struct pv_page *pvpage; 496 struct pv_entry *pv; 497 498 simple_lock(&pvalloc_lock); 499 500 if (pv_freepages.tqh_first != NULL) { 501 pvpage = pv_freepages.tqh_first; 502 pvpage->pvinfo.pvpi_nfree--; 503 if (pvpage->pvinfo.pvpi_nfree == 0) { 504 /* nothing left in this one? */ 505 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 506 } 507 pv = pvpage->pvinfo.pvpi_pvfree; 508 #ifdef DIAGNOSTIC 509 if (pv == NULL) 510 panic("pmap_alloc_pv: pvpi_nfree off"); 511 #endif 512 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 513 pv_nfpvents--; /* took one from pool */ 514 } else { 515 pv = NULL; /* need more of them */ 516 } 517 518 /* 519 * if below low water mark or we didn't get a pv_entry we try and 520 * create more pv_entrys ... 521 */ 522 523 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 524 if (pv == NULL) 525 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 526 mode : ALLOCPV_NEED); 527 else 528 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 529 } 530 531 simple_unlock(&pvalloc_lock); 532 return(pv); 533 } 534 535 /* 536 * pmap_alloc_pvpage: maybe allocate a new pvpage 537 * 538 * if need_entry is false: try and allocate a new pv_page 539 * if need_entry is true: try and allocate a new pv_page and return a 540 * new pv_entry from it. if we are unable to allocate a pv_page 541 * we make a last ditch effort to steal a pv_page from some other 542 * mapping. if that fails, we panic... 543 * 544 * => we assume that the caller holds pvalloc_lock 545 */ 546 547 static struct pv_entry * 548 pmap_alloc_pvpage(pmap, mode) 549 struct pmap *pmap; 550 int mode; 551 { 552 struct vm_page *pg; 553 struct pv_page *pvpage; 554 struct pv_entry *pv; 555 int s; 556 557 /* 558 * if we need_entry and we've got unused pv_pages, allocate from there 559 */ 560 561 if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) { 562 563 /* move it to pv_freepages list */ 564 pvpage = pv_unusedpgs.tqh_first; 565 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 566 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 567 568 /* allocate a pv_entry */ 569 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 570 pv = pvpage->pvinfo.pvpi_pvfree; 571 #ifdef DIAGNOSTIC 572 if (pv == NULL) 573 panic("pmap_alloc_pvpage: pvpi_nfree off"); 574 #endif 575 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 576 577 pv_nfpvents--; /* took one from pool */ 578 return(pv); 579 } 580 581 /* 582 * see if we've got a cached unmapped VA that we can map a page in. 583 * if not, try to allocate one. 584 */ 585 586 587 if (pv_cachedva == 0) { 588 s = splvm(); 589 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 590 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 591 splx(s); 592 if (pv_cachedva == 0) { 593 return (NULL); 594 } 595 } 596 597 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 598 UVM_PGA_USERESERVE); 599 if (pg) 600 pg->flags &= ~PG_BUSY; /* never busy */ 601 602 if (pg == NULL) 603 return (NULL); 604 605 /* 606 * add a mapping for our new pv_page and free its entrys (save one!) 607 * 608 * NOTE: If we are allocating a PV page for the kernel pmap, the 609 * pmap is already locked! (...but entering the mapping is safe...) 610 */ 611 612 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL); 613 pmap_update(pmap_kernel()); 614 pvpage = (struct pv_page *) pv_cachedva; 615 pv_cachedva = 0; 616 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 617 } 618 619 /* 620 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 621 * 622 * => caller must hold pvalloc_lock 623 * => if need_entry is true, we allocate and return one pv_entry 624 */ 625 626 static struct pv_entry * 627 pmap_add_pvpage(pvp, need_entry) 628 struct pv_page *pvp; 629 boolean_t need_entry; 630 { 631 int tofree, lcv; 632 633 /* do we need to return one? */ 634 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 635 636 pvp->pvinfo.pvpi_pvfree = NULL; 637 pvp->pvinfo.pvpi_nfree = tofree; 638 for (lcv = 0 ; lcv < tofree ; lcv++) { 639 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 640 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 641 } 642 if (need_entry) 643 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 644 else 645 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 646 pv_nfpvents += tofree; 647 return((need_entry) ? &pvp->pvents[lcv] : NULL); 648 } 649 650 /* 651 * pmap_free_pv_doit: actually free a pv_entry 652 * 653 * => do not call this directly! instead use either 654 * 1. pmap_free_pv ==> free a single pv_entry 655 * 2. pmap_free_pvs => free a list of pv_entrys 656 * => we must be holding pvalloc_lock 657 */ 658 659 __inline static void 660 pmap_free_pv_doit(pv) 661 struct pv_entry *pv; 662 { 663 struct pv_page *pvp; 664 665 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 666 pv_nfpvents++; 667 pvp->pvinfo.pvpi_nfree++; 668 669 /* nfree == 1 => fully allocated page just became partly allocated */ 670 if (pvp->pvinfo.pvpi_nfree == 1) { 671 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 672 } 673 674 /* free it */ 675 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 676 pvp->pvinfo.pvpi_pvfree = pv; 677 678 /* 679 * are all pv_page's pv_entry's free? move it to unused queue. 680 */ 681 682 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 683 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 684 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 685 } 686 } 687 688 /* 689 * pmap_free_pv: free a single pv_entry 690 * 691 * => we gain the pvalloc_lock 692 */ 693 694 __inline static void 695 pmap_free_pv(pmap, pv) 696 struct pmap *pmap; 697 struct pv_entry *pv; 698 { 699 simple_lock(&pvalloc_lock); 700 pmap_free_pv_doit(pv); 701 702 /* 703 * Can't free the PV page if the PV entries were associated with 704 * the kernel pmap; the pmap is already locked. 705 */ 706 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 707 pmap != pmap_kernel()) 708 pmap_free_pvpage(); 709 710 simple_unlock(&pvalloc_lock); 711 } 712 713 /* 714 * pmap_free_pvs: free a list of pv_entrys 715 * 716 * => we gain the pvalloc_lock 717 */ 718 719 __inline static void 720 pmap_free_pvs(pmap, pvs) 721 struct pmap *pmap; 722 struct pv_entry *pvs; 723 { 724 struct pv_entry *nextpv; 725 726 simple_lock(&pvalloc_lock); 727 728 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 729 nextpv = pvs->pv_next; 730 pmap_free_pv_doit(pvs); 731 } 732 733 /* 734 * Can't free the PV page if the PV entries were associated with 735 * the kernel pmap; the pmap is already locked. 736 */ 737 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 738 pmap != pmap_kernel()) 739 pmap_free_pvpage(); 740 741 simple_unlock(&pvalloc_lock); 742 } 743 744 745 /* 746 * pmap_free_pvpage: try and free an unused pv_page structure 747 * 748 * => assume caller is holding the pvalloc_lock and that 749 * there is a page on the pv_unusedpgs list 750 * => if we can't get a lock on the kmem_map we try again later 751 * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows 752 * that if we can lock the kmem_map then we are not already 753 * holding kmem_object's lock. 754 */ 755 756 static void 757 pmap_free_pvpage() 758 { 759 int s; 760 struct vm_map *map; 761 struct vm_map_entry *dead_entries; 762 struct pv_page *pvp; 763 764 s = splvm(); /* protect kmem_map */ 765 766 pvp = pv_unusedpgs.tqh_first; 767 768 /* 769 * note: watch out for pv_initpage which is allocated out of 770 * kernel_map rather than kmem_map. 771 */ 772 if (pvp == pv_initpage) 773 map = kernel_map; 774 else 775 map = kmem_map; 776 777 if (vm_map_lock_try(map)) { 778 779 /* remove pvp from pv_unusedpgs */ 780 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 781 782 /* unmap the page */ 783 dead_entries = NULL; 784 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 785 &dead_entries); 786 vm_map_unlock(map); 787 788 if (dead_entries != NULL) 789 uvm_unmap_detach(dead_entries, 0); 790 791 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 792 } 793 794 if (pvp == pv_initpage) 795 /* no more initpage, we've freed it */ 796 pv_initpage = NULL; 797 798 splx(s); 799 } 800 801 /* 802 * main pv_entry manipulation functions: 803 * pmap_enter_pv: enter a mapping onto a pv_head list 804 * pmap_remove_pv: remove a mappiing from a pv_head list 805 * 806 * NOTE: pmap_enter_pv expects to lock the pvh itself 807 * pmap_remove_pv expects te caller to lock the pvh before calling 808 */ 809 810 /* 811 * pmap_enter_pv: enter a mapping onto a pv_head lst 812 * 813 * => caller should hold the proper lock on pmap_main_lock 814 * => caller should have pmap locked 815 * => we will gain the lock on the pv_head and allocate the new pv_entry 816 * => caller should adjust ptp's wire_count before calling 817 * => caller should not adjust pmap's wire_count 818 */ 819 820 __inline static void 821 pmap_enter_pv(pvh, pve, pmap, va, ptp, flags) 822 struct pv_head *pvh; 823 struct pv_entry *pve; /* preallocated pve for us to use */ 824 struct pmap *pmap; 825 vaddr_t va; 826 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 827 int flags; 828 { 829 pve->pv_pmap = pmap; 830 pve->pv_va = va; 831 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 832 pve->pv_flags = flags; 833 simple_lock(&pvh->pvh_lock); /* lock pv_head */ 834 pve->pv_next = pvh->pvh_list; /* add to ... */ 835 pvh->pvh_list = pve; /* ... locked list */ 836 simple_unlock(&pvh->pvh_lock); /* unlock, done! */ 837 if (pve->pv_flags & PT_W) 838 ++pmap->pm_stats.wired_count; 839 } 840 841 /* 842 * pmap_remove_pv: try to remove a mapping from a pv_list 843 * 844 * => caller should hold proper lock on pmap_main_lock 845 * => pmap should be locked 846 * => caller should hold lock on pv_head [so that attrs can be adjusted] 847 * => caller should adjust ptp's wire_count and free PTP if needed 848 * => caller should NOT adjust pmap's wire_count 849 * => we return the removed pve 850 */ 851 852 __inline static struct pv_entry * 853 pmap_remove_pv(pvh, pmap, va) 854 struct pv_head *pvh; 855 struct pmap *pmap; 856 vaddr_t va; 857 { 858 struct pv_entry *pve, **prevptr; 859 860 prevptr = &pvh->pvh_list; /* previous pv_entry pointer */ 861 pve = *prevptr; 862 while (pve) { 863 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 864 *prevptr = pve->pv_next; /* remove it! */ 865 if (pve->pv_flags & PT_W) 866 --pmap->pm_stats.wired_count; 867 break; 868 } 869 prevptr = &pve->pv_next; /* previous pointer */ 870 pve = pve->pv_next; /* advance */ 871 } 872 return(pve); /* return removed pve */ 873 } 874 875 /* 876 * 877 * pmap_modify_pv: Update pv flags 878 * 879 * => caller should hold lock on pv_head [so that attrs can be adjusted] 880 * => caller should NOT adjust pmap's wire_count 881 * => caller must call pmap_vac_me_harder() if writable status of a page 882 * may have changed. 883 * => we return the old flags 884 * 885 * Modify a physical-virtual mapping in the pv table 886 */ 887 888 /*__inline */ 889 static u_int 890 pmap_modify_pv(pmap, va, pvh, bic_mask, eor_mask) 891 struct pmap *pmap; 892 vaddr_t va; 893 struct pv_head *pvh; 894 u_int bic_mask; 895 u_int eor_mask; 896 { 897 struct pv_entry *npv; 898 u_int flags, oflags; 899 900 /* 901 * There is at least one VA mapping this page. 902 */ 903 904 for (npv = pvh->pvh_list; npv; npv = npv->pv_next) { 905 if (pmap == npv->pv_pmap && va == npv->pv_va) { 906 oflags = npv->pv_flags; 907 npv->pv_flags = flags = 908 ((oflags & ~bic_mask) ^ eor_mask); 909 if ((flags ^ oflags) & PT_W) { 910 if (flags & PT_W) 911 ++pmap->pm_stats.wired_count; 912 else 913 --pmap->pm_stats.wired_count; 914 } 915 return (oflags); 916 } 917 } 918 return (0); 919 } 920 921 /* 922 * Map the specified level 2 pagetable into the level 1 page table for 923 * the given pmap to cover a chunk of virtual address space starting from the 924 * address specified. 925 */ 926 static /*__inline*/ void 927 pmap_map_in_l1(pmap, va, l2pa, selfref) 928 struct pmap *pmap; 929 vaddr_t va, l2pa; 930 boolean_t selfref; 931 { 932 vaddr_t ptva; 933 934 /* Calculate the index into the L1 page table. */ 935 ptva = (va >> PDSHIFT) & ~3; 936 937 PDEBUG(0, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 938 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 939 940 /* Map page table into the L1. */ 941 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 942 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 943 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 944 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 945 946 PDEBUG(0, printf("pt self reference %lx in %lx\n", 947 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 948 949 /* Map the page table into the page table area. */ 950 if (selfref) { 951 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 952 L2_PTE_NC_NB(l2pa, AP_KRW); 953 } 954 /* XXX should be a purge */ 955 /* cpu_tlb_flushD();*/ 956 } 957 958 #if 0 959 static /*__inline*/ void 960 pmap_unmap_in_l1(pmap, va) 961 struct pmap *pmap; 962 vaddr_t va; 963 { 964 vaddr_t ptva; 965 966 /* Calculate the index into the L1 page table. */ 967 ptva = (va >> PDSHIFT) & ~3; 968 969 /* Unmap page table from the L1. */ 970 pmap->pm_pdir[ptva + 0] = 0; 971 pmap->pm_pdir[ptva + 1] = 0; 972 pmap->pm_pdir[ptva + 2] = 0; 973 pmap->pm_pdir[ptva + 3] = 0; 974 975 /* Unmap the page table from the page table area. */ 976 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 977 978 /* XXX should be a purge */ 979 /* cpu_tlb_flushD();*/ 980 } 981 #endif 982 983 /* 984 * Used to map a range of physical addresses into kernel 985 * virtual address space. 986 * 987 * For now, VM is already on, we only need to map the 988 * specified memory. 989 */ 990 vaddr_t 991 pmap_map(va, spa, epa, prot) 992 vaddr_t va, spa, epa; 993 int prot; 994 { 995 while (spa < epa) { 996 pmap_kenter_pa(va, spa, prot); 997 va += NBPG; 998 spa += NBPG; 999 } 1000 pmap_update(pmap_kernel()); 1001 return(va); 1002 } 1003 1004 1005 /* 1006 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1007 * 1008 * bootstrap the pmap system. This is called from initarm and allows 1009 * the pmap system to initailise any structures it requires. 1010 * 1011 * Currently this sets up the kernel_pmap that is statically allocated 1012 * and also allocated virtual addresses for certain page hooks. 1013 * Currently the only one page hook is allocated that is used 1014 * to zero physical pages of memory. 1015 * It also initialises the start and end address of the kernel data space. 1016 */ 1017 extern paddr_t physical_freestart; 1018 extern paddr_t physical_freeend; 1019 1020 char *boot_head; 1021 1022 void 1023 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1024 pd_entry_t *kernel_l1pt; 1025 pv_addr_t kernel_ptpt; 1026 { 1027 int loop; 1028 paddr_t start, end; 1029 #if NISADMA > 0 1030 paddr_t istart; 1031 psize_t isize; 1032 #endif 1033 1034 pmap_kernel()->pm_pdir = kernel_l1pt; 1035 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1036 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1037 simple_lock_init(&pmap_kernel()->pm_lock); 1038 pmap_kernel()->pm_obj.pgops = NULL; 1039 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1040 pmap_kernel()->pm_obj.uo_npages = 0; 1041 pmap_kernel()->pm_obj.uo_refs = 1; 1042 1043 /* 1044 * Initialize PAGE_SIZE-dependent variables. 1045 */ 1046 uvm_setpagesize(); 1047 1048 npages = 0; 1049 loop = 0; 1050 while (loop < bootconfig.dramblocks) { 1051 start = (paddr_t)bootconfig.dram[loop].address; 1052 end = start + (bootconfig.dram[loop].pages * NBPG); 1053 if (start < physical_freestart) 1054 start = physical_freestart; 1055 if (end > physical_freeend) 1056 end = physical_freeend; 1057 #if 0 1058 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1059 #endif 1060 #if NISADMA > 0 1061 if (pmap_isa_dma_range_intersect(start, end - start, 1062 &istart, &isize)) { 1063 /* 1064 * Place the pages that intersect with the 1065 * ISA DMA range onto the ISA DMA free list. 1066 */ 1067 #if 0 1068 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1069 istart + isize - 1); 1070 #endif 1071 uvm_page_physload(atop(istart), 1072 atop(istart + isize), atop(istart), 1073 atop(istart + isize), VM_FREELIST_ISADMA); 1074 npages += atop(istart + isize) - atop(istart); 1075 1076 /* 1077 * Load the pieces that come before 1078 * the intersection into the default 1079 * free list. 1080 */ 1081 if (start < istart) { 1082 #if 0 1083 printf(" BEFORE 0x%lx -> 0x%lx\n", 1084 start, istart - 1); 1085 #endif 1086 uvm_page_physload(atop(start), 1087 atop(istart), atop(start), 1088 atop(istart), VM_FREELIST_DEFAULT); 1089 npages += atop(istart) - atop(start); 1090 } 1091 1092 /* 1093 * Load the pieces that come after 1094 * the intersection into the default 1095 * free list. 1096 */ 1097 if ((istart + isize) < end) { 1098 #if 0 1099 printf(" AFTER 0x%lx -> 0x%lx\n", 1100 (istart + isize), end - 1); 1101 #endif 1102 uvm_page_physload(atop(istart + isize), 1103 atop(end), atop(istart + isize), 1104 atop(end), VM_FREELIST_DEFAULT); 1105 npages += atop(end) - atop(istart + isize); 1106 } 1107 } else { 1108 uvm_page_physload(atop(start), atop(end), 1109 atop(start), atop(end), VM_FREELIST_DEFAULT); 1110 npages += atop(end) - atop(start); 1111 } 1112 #else /* NISADMA > 0 */ 1113 uvm_page_physload(atop(start), atop(end), 1114 atop(start), atop(end), VM_FREELIST_DEFAULT); 1115 npages += atop(end) - atop(start); 1116 #endif /* NISADMA > 0 */ 1117 ++loop; 1118 } 1119 1120 #ifdef MYCROFT_HACK 1121 printf("npages = %ld\n", npages); 1122 #endif 1123 1124 virtual_start = KERNEL_VM_BASE; 1125 virtual_end = virtual_start + KERNEL_VM_SIZE - 1; 1126 1127 ALLOC_PAGE_HOOK(page_hook0, NBPG); 1128 ALLOC_PAGE_HOOK(page_hook1, NBPG); 1129 1130 /* 1131 * The mem special device needs a virtual hook but we don't 1132 * need a pte 1133 */ 1134 memhook = (char *)virtual_start; 1135 virtual_start += NBPG; 1136 1137 msgbufaddr = (caddr_t)virtual_start; 1138 msgbufpte = (pt_entry_t)pmap_pte(pmap_kernel(), virtual_start); 1139 virtual_start += round_page(MSGBUFSIZE); 1140 1141 /* 1142 * init the static-global locks and global lists. 1143 */ 1144 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1145 simple_lock_init(&pvalloc_lock); 1146 TAILQ_INIT(&pv_freepages); 1147 TAILQ_INIT(&pv_unusedpgs); 1148 1149 /* 1150 * compute the number of pages we have and then allocate RAM 1151 * for each pages' pv_head and saved attributes. 1152 */ 1153 { 1154 int npages, lcv; 1155 vsize_t s; 1156 1157 npages = 0; 1158 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) 1159 npages += (vm_physmem[lcv].end - vm_physmem[lcv].start); 1160 s = (vsize_t) (sizeof(struct pv_head) * npages + 1161 sizeof(char) * npages); 1162 s = round_page(s); /* round up */ 1163 boot_head = (char *)uvm_pageboot_alloc(s); 1164 bzero((char *)boot_head, s); 1165 if (boot_head == 0) 1166 panic("pmap_init: unable to allocate pv_heads"); 1167 } 1168 1169 /* 1170 * initialize the pmap pool. 1171 */ 1172 1173 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1174 0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP); 1175 1176 cpu_dcache_wbinv_all(); 1177 } 1178 1179 /* 1180 * void pmap_init(void) 1181 * 1182 * Initialize the pmap module. 1183 * Called by vm_init() in vm/vm_init.c in order to initialise 1184 * any structures that the pmap system needs to map virtual memory. 1185 */ 1186 1187 extern int physmem; 1188 1189 void 1190 pmap_init() 1191 { 1192 int lcv, i; 1193 1194 #ifdef MYCROFT_HACK 1195 printf("physmem = %d\n", physmem); 1196 #endif 1197 1198 /* 1199 * Set the available memory vars - These do not map to real memory 1200 * addresses and cannot as the physical memory is fragmented. 1201 * They are used by ps for %mem calculations. 1202 * One could argue whether this should be the entire memory or just 1203 * the memory that is useable in a user process. 1204 */ 1205 avail_start = 0; 1206 avail_end = physmem * NBPG; 1207 1208 /* allocate pv_head stuff first */ 1209 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1210 vm_physmem[lcv].pmseg.pvhead = (struct pv_head *)boot_head; 1211 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.pvhead + 1212 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1213 for (i = 0; 1214 i < (vm_physmem[lcv].end - vm_physmem[lcv].start); i++) { 1215 simple_lock_init( 1216 &vm_physmem[lcv].pmseg.pvhead[i].pvh_lock); 1217 } 1218 } 1219 1220 /* now allocate attrs */ 1221 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1222 vm_physmem[lcv].pmseg.attrs = (char *) boot_head; 1223 boot_head = (char *)(vaddr_t)(vm_physmem[lcv].pmseg.attrs + 1224 (vm_physmem[lcv].end - vm_physmem[lcv].start)); 1225 } 1226 1227 /* 1228 * now we need to free enough pv_entry structures to allow us to get 1229 * the kmem_map/kmem_object allocated and inited (done after this 1230 * function is finished). to do this we allocate one bootstrap page out 1231 * of kernel_map and use it to provide an initial pool of pv_entry 1232 * structures. we never free this page. 1233 */ 1234 1235 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1236 if (pv_initpage == NULL) 1237 panic("pmap_init: pv_initpage"); 1238 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1239 pv_nfpvents = 0; 1240 (void) pmap_add_pvpage(pv_initpage, FALSE); 1241 1242 #ifdef MYCROFT_HACK 1243 for (lcv = 0 ; lcv < vm_nphysseg ; lcv++) { 1244 printf("physseg[%d] pvent=%p attrs=%p start=%ld end=%ld\n", 1245 lcv, 1246 vm_physmem[lcv].pmseg.pvent, vm_physmem[lcv].pmseg.attrs, 1247 vm_physmem[lcv].start, vm_physmem[lcv].end); 1248 } 1249 #endif 1250 pmap_initialized = TRUE; 1251 1252 /* Initialise our L1 page table queues and counters */ 1253 SIMPLEQ_INIT(&l1pt_static_queue); 1254 l1pt_static_queue_count = 0; 1255 l1pt_static_create_count = 0; 1256 SIMPLEQ_INIT(&l1pt_queue); 1257 l1pt_queue_count = 0; 1258 l1pt_create_count = 0; 1259 l1pt_reuse_count = 0; 1260 } 1261 1262 /* 1263 * pmap_postinit() 1264 * 1265 * This routine is called after the vm and kmem subsystems have been 1266 * initialised. This allows the pmap code to perform any initialisation 1267 * that can only be done one the memory allocation is in place. 1268 */ 1269 1270 void 1271 pmap_postinit() 1272 { 1273 int loop; 1274 struct l1pt *pt; 1275 1276 #ifdef PMAP_STATIC_L1S 1277 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1278 #else /* PMAP_STATIC_L1S */ 1279 for (loop = 0; loop < max_processes; ++loop) { 1280 #endif /* PMAP_STATIC_L1S */ 1281 /* Allocate a L1 page table */ 1282 pt = pmap_alloc_l1pt(); 1283 if (!pt) 1284 panic("Cannot allocate static L1 page tables\n"); 1285 1286 /* Clean it */ 1287 bzero((void *)pt->pt_va, PD_SIZE); 1288 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1289 /* Add the page table to the queue */ 1290 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1291 ++l1pt_static_queue_count; 1292 ++l1pt_static_create_count; 1293 } 1294 } 1295 1296 1297 /* 1298 * Create and return a physical map. 1299 * 1300 * If the size specified for the map is zero, the map is an actual physical 1301 * map, and may be referenced by the hardware. 1302 * 1303 * If the size specified is non-zero, the map will be used in software only, 1304 * and is bounded by that size. 1305 */ 1306 1307 pmap_t 1308 pmap_create() 1309 { 1310 struct pmap *pmap; 1311 1312 /* 1313 * Fetch pmap entry from the pool 1314 */ 1315 1316 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1317 /* XXX is this really needed! */ 1318 memset(pmap, 0, sizeof(*pmap)); 1319 1320 simple_lock_init(&pmap->pm_obj.vmobjlock); 1321 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1322 TAILQ_INIT(&pmap->pm_obj.memq); 1323 pmap->pm_obj.uo_npages = 0; 1324 pmap->pm_obj.uo_refs = 1; 1325 pmap->pm_stats.wired_count = 0; 1326 pmap->pm_stats.resident_count = 1; 1327 1328 /* Now init the machine part of the pmap */ 1329 pmap_pinit(pmap); 1330 return(pmap); 1331 } 1332 1333 /* 1334 * pmap_alloc_l1pt() 1335 * 1336 * This routine allocates physical and virtual memory for a L1 page table 1337 * and wires it. 1338 * A l1pt structure is returned to describe the allocated page table. 1339 * 1340 * This routine is allowed to fail if the required memory cannot be allocated. 1341 * In this case NULL is returned. 1342 */ 1343 1344 struct l1pt * 1345 pmap_alloc_l1pt(void) 1346 { 1347 paddr_t pa; 1348 vaddr_t va; 1349 struct l1pt *pt; 1350 int error; 1351 struct vm_page *m; 1352 pt_entry_t *ptes; 1353 1354 /* Allocate virtual address space for the L1 page table */ 1355 va = uvm_km_valloc(kernel_map, PD_SIZE); 1356 if (va == 0) { 1357 #ifdef DIAGNOSTIC 1358 PDEBUG(0, 1359 printf("pmap: Cannot allocate pageable memory for L1\n")); 1360 #endif /* DIAGNOSTIC */ 1361 return(NULL); 1362 } 1363 1364 /* Allocate memory for the l1pt structure */ 1365 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1366 1367 /* 1368 * Allocate pages from the VM system. 1369 */ 1370 TAILQ_INIT(&pt->pt_plist); 1371 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1372 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1373 if (error) { 1374 #ifdef DIAGNOSTIC 1375 PDEBUG(0, 1376 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1377 error)); 1378 #endif /* DIAGNOSTIC */ 1379 /* Release the resources we already have claimed */ 1380 free(pt, M_VMPMAP); 1381 uvm_km_free(kernel_map, va, PD_SIZE); 1382 return(NULL); 1383 } 1384 1385 /* Map our physical pages into our virtual space */ 1386 pt->pt_va = va; 1387 m = pt->pt_plist.tqh_first; 1388 ptes = pmap_map_ptes(pmap_kernel()); 1389 while (m && va < (pt->pt_va + PD_SIZE)) { 1390 pa = VM_PAGE_TO_PHYS(m); 1391 1392 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1393 1394 /* Revoke cacheability and bufferability */ 1395 /* XXX should be done better than this */ 1396 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1397 1398 va += NBPG; 1399 m = m->pageq.tqe_next; 1400 } 1401 pmap_unmap_ptes(pmap_kernel()); 1402 pmap_update(pmap_kernel()); 1403 1404 #ifdef DIAGNOSTIC 1405 if (m) 1406 panic("pmap_alloc_l1pt: pglist not empty\n"); 1407 #endif /* DIAGNOSTIC */ 1408 1409 pt->pt_flags = 0; 1410 return(pt); 1411 } 1412 1413 /* 1414 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1415 */ 1416 static void 1417 pmap_free_l1pt(pt) 1418 struct l1pt *pt; 1419 { 1420 /* Separate the physical memory for the virtual space */ 1421 pmap_kremove(pt->pt_va, PD_SIZE); 1422 pmap_update(pmap_kernel()); 1423 1424 /* Return the physical memory */ 1425 uvm_pglistfree(&pt->pt_plist); 1426 1427 /* Free the virtual space */ 1428 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1429 1430 /* Free the l1pt structure */ 1431 free(pt, M_VMPMAP); 1432 } 1433 1434 /* 1435 * Allocate a page directory. 1436 * This routine will either allocate a new page directory from the pool 1437 * of L1 page tables currently held by the kernel or it will allocate 1438 * a new one via pmap_alloc_l1pt(). 1439 * It will then initialise the l1 page table for use. 1440 */ 1441 static int 1442 pmap_allocpagedir(pmap) 1443 struct pmap *pmap; 1444 { 1445 paddr_t pa; 1446 struct l1pt *pt; 1447 pt_entry_t *pte; 1448 1449 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1450 1451 /* Do we have any spare L1's lying around ? */ 1452 if (l1pt_static_queue_count) { 1453 --l1pt_static_queue_count; 1454 pt = l1pt_static_queue.sqh_first; 1455 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1456 } else if (l1pt_queue_count) { 1457 --l1pt_queue_count; 1458 pt = l1pt_queue.sqh_first; 1459 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1460 ++l1pt_reuse_count; 1461 } else { 1462 pt = pmap_alloc_l1pt(); 1463 if (!pt) 1464 return(ENOMEM); 1465 ++l1pt_create_count; 1466 } 1467 1468 /* Store the pointer to the l1 descriptor in the pmap. */ 1469 pmap->pm_l1pt = pt; 1470 1471 /* Get the physical address of the start of the l1 */ 1472 pa = VM_PAGE_TO_PHYS(pt->pt_plist.tqh_first); 1473 1474 /* Store the virtual address of the l1 in the pmap. */ 1475 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1476 1477 /* Clean the L1 if it is dirty */ 1478 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1479 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1480 1481 /* Do we already have the kernel mappings ? */ 1482 if (!(pt->pt_flags & PTFLAG_KPT)) { 1483 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1484 1485 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1486 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1487 KERNEL_PD_SIZE); 1488 pt->pt_flags |= PTFLAG_KPT; 1489 } 1490 1491 /* Allocate a page table to map all the page tables for this pmap */ 1492 1493 #ifdef DIAGNOSTIC 1494 if (pmap->pm_vptpt) { 1495 /* XXX What if we have one already ? */ 1496 panic("pmap_allocpagedir: have pt already\n"); 1497 } 1498 #endif /* DIAGNOSTIC */ 1499 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1500 if (pmap->pm_vptpt == 0) { 1501 pmap_freepagedir(pmap); 1502 return(ENOMEM); 1503 } 1504 1505 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1506 pmap->pm_pptpt &= PG_FRAME; 1507 /* Revoke cacheability and bufferability */ 1508 /* XXX should be done better than this */ 1509 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1510 *pte = *pte & ~(PT_C | PT_B); 1511 1512 /* Wire in this page table */ 1513 pmap_map_in_l1(pmap, PROCESS_PAGE_TBLS_BASE, pmap->pm_pptpt, TRUE); 1514 1515 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1516 1517 /* 1518 * Map the kernel page tables for 0xf0000000 + 1519 * into the page table used to map the 1520 * pmap's page tables 1521 */ 1522 bcopy((char *)(PROCESS_PAGE_TBLS_BASE 1523 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) 1524 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1525 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1526 (KERNEL_PD_SIZE >> 2)); 1527 1528 return(0); 1529 } 1530 1531 1532 /* 1533 * Initialize a preallocated and zeroed pmap structure, 1534 * such as one in a vmspace structure. 1535 */ 1536 1537 void 1538 pmap_pinit(pmap) 1539 struct pmap *pmap; 1540 { 1541 int backoff = 6; 1542 int retry = 10; 1543 1544 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1545 1546 /* Keep looping until we succeed in allocating a page directory */ 1547 while (pmap_allocpagedir(pmap) != 0) { 1548 /* 1549 * Ok we failed to allocate a suitable block of memory for an 1550 * L1 page table. This means that either: 1551 * 1. 16KB of virtual address space could not be allocated 1552 * 2. 16KB of physically contiguous memory on a 16KB boundary 1553 * could not be allocated. 1554 * 1555 * Since we cannot fail we will sleep for a while and try 1556 * again. 1557 * 1558 * Searching for a suitable L1 PT is expensive: 1559 * to avoid hogging the system when memory is really 1560 * scarce, use an exponential back-off so that 1561 * eventually we won't retry more than once every 8 1562 * seconds. This should allow other processes to run 1563 * to completion and free up resources. 1564 */ 1565 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1566 NULL); 1567 if (--retry == 0) { 1568 retry = 10; 1569 if (backoff) 1570 --backoff; 1571 } 1572 } 1573 1574 /* Map zero page for the pmap. This will also map the L2 for it */ 1575 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1576 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1577 pmap_update(pmap); 1578 } 1579 1580 1581 void 1582 pmap_freepagedir(pmap) 1583 struct pmap *pmap; 1584 { 1585 /* Free the memory used for the page table mapping */ 1586 if (pmap->pm_vptpt != 0) 1587 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1588 1589 /* junk the L1 page table */ 1590 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1591 /* Add the page table to the queue */ 1592 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1593 ++l1pt_static_queue_count; 1594 } else if (l1pt_queue_count < 8) { 1595 /* Add the page table to the queue */ 1596 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1597 ++l1pt_queue_count; 1598 } else 1599 pmap_free_l1pt(pmap->pm_l1pt); 1600 } 1601 1602 1603 /* 1604 * Retire the given physical map from service. 1605 * Should only be called if the map contains no valid mappings. 1606 */ 1607 1608 void 1609 pmap_destroy(pmap) 1610 struct pmap *pmap; 1611 { 1612 struct vm_page *page; 1613 int count; 1614 1615 if (pmap == NULL) 1616 return; 1617 1618 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1619 1620 /* 1621 * Drop reference count 1622 */ 1623 simple_lock(&pmap->pm_obj.vmobjlock); 1624 count = --pmap->pm_obj.uo_refs; 1625 simple_unlock(&pmap->pm_obj.vmobjlock); 1626 if (count > 0) { 1627 return; 1628 } 1629 1630 /* 1631 * reference count is zero, free pmap resources and then free pmap. 1632 */ 1633 1634 /* Remove the zero page mapping */ 1635 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1636 pmap_update(pmap); 1637 1638 /* 1639 * Free any page tables still mapped 1640 * This is only temporay until pmap_enter can count the number 1641 * of mappings made in a page table. Then pmap_remove() can 1642 * reduce the count and free the pagetable when the count 1643 * reaches zero. Note that entries in this list should match the 1644 * contents of the ptpt, however this is faster than walking a 1024 1645 * entries looking for pt's 1646 * taken from i386 pmap.c 1647 */ 1648 while (pmap->pm_obj.memq.tqh_first != NULL) { 1649 page = pmap->pm_obj.memq.tqh_first; 1650 #ifdef DIAGNOSTIC 1651 if (page->flags & PG_BUSY) 1652 panic("pmap_release: busy page table page"); 1653 #endif 1654 /* pmap_page_protect? currently no need for it. */ 1655 1656 page->wire_count = 0; 1657 uvm_pagefree(page); 1658 } 1659 1660 /* Free the page dir */ 1661 pmap_freepagedir(pmap); 1662 1663 /* return the pmap to the pool */ 1664 pool_put(&pmap_pmap_pool, pmap); 1665 } 1666 1667 1668 /* 1669 * void pmap_reference(struct pmap *pmap) 1670 * 1671 * Add a reference to the specified pmap. 1672 */ 1673 1674 void 1675 pmap_reference(pmap) 1676 struct pmap *pmap; 1677 { 1678 if (pmap == NULL) 1679 return; 1680 1681 simple_lock(&pmap->pm_lock); 1682 pmap->pm_obj.uo_refs++; 1683 simple_unlock(&pmap->pm_lock); 1684 } 1685 1686 /* 1687 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1688 * 1689 * Return the start and end addresses of the kernel's virtual space. 1690 * These values are setup in pmap_bootstrap and are updated as pages 1691 * are allocated. 1692 */ 1693 1694 void 1695 pmap_virtual_space(start, end) 1696 vaddr_t *start; 1697 vaddr_t *end; 1698 { 1699 *start = virtual_start; 1700 *end = virtual_end; 1701 } 1702 1703 1704 /* 1705 * Activate the address space for the specified process. If the process 1706 * is the current process, load the new MMU context. 1707 */ 1708 void 1709 pmap_activate(p) 1710 struct proc *p; 1711 { 1712 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1713 struct pcb *pcb = &p->p_addr->u_pcb; 1714 1715 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1716 (paddr_t *)&pcb->pcb_pagedir); 1717 1718 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1719 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1720 1721 if (p == curproc) { 1722 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1723 setttb((u_int)pcb->pcb_pagedir); 1724 } 1725 #if 0 1726 pmap->pm_pdchanged = FALSE; 1727 #endif 1728 } 1729 1730 1731 /* 1732 * Deactivate the address space of the specified process. 1733 */ 1734 void 1735 pmap_deactivate(p) 1736 struct proc *p; 1737 { 1738 } 1739 1740 /* 1741 * Perform any deferred pmap operations. 1742 */ 1743 void 1744 pmap_update(struct pmap *pmap) 1745 { 1746 1747 /* 1748 * We haven't deferred any pmap operations, but we do need to 1749 * make sure TLB/cache operations have completed. 1750 */ 1751 cpu_cpwait(); 1752 } 1753 1754 /* 1755 * pmap_clean_page() 1756 * 1757 * This is a local function used to work out the best strategy to clean 1758 * a single page referenced by its entry in the PV table. It's used by 1759 * pmap_copy_page, pmap_zero page and maybe some others later on. 1760 * 1761 * Its policy is effectively: 1762 * o If there are no mappings, we don't bother doing anything with the cache. 1763 * o If there is one mapping, we clean just that page. 1764 * o If there are multiple mappings, we clean the entire cache. 1765 * 1766 * So that some functions can be further optimised, it returns 0 if it didn't 1767 * clean the entire cache, or 1 if it did. 1768 * 1769 * XXX One bug in this routine is that if the pv_entry has a single page 1770 * mapped at 0x00000000 a whole cache clean will be performed rather than 1771 * just the 1 page. Since this should not occur in everyday use and if it does 1772 * it will just result in not the most efficient clean for the page. 1773 */ 1774 static int 1775 pmap_clean_page(pv, is_src) 1776 struct pv_entry *pv; 1777 boolean_t is_src; 1778 { 1779 struct pmap *pmap; 1780 struct pv_entry *npv; 1781 int cache_needs_cleaning = 0; 1782 vaddr_t page_to_clean = 0; 1783 1784 if (pv == NULL) 1785 /* nothing mapped in so nothing to flush */ 1786 return (0); 1787 1788 /* Since we flush the cache each time we change curproc, we 1789 * only need to flush the page if it is in the current pmap. 1790 */ 1791 if (curproc) 1792 pmap = curproc->p_vmspace->vm_map.pmap; 1793 else 1794 pmap = pmap_kernel(); 1795 1796 for (npv = pv; npv; npv = npv->pv_next) { 1797 if (npv->pv_pmap == pmap) { 1798 /* The page is mapped non-cacheable in 1799 * this map. No need to flush the cache. 1800 */ 1801 if (npv->pv_flags & PT_NC) { 1802 #ifdef DIAGNOSTIC 1803 if (cache_needs_cleaning) 1804 panic("pmap_clean_page: " 1805 "cache inconsistency"); 1806 #endif 1807 break; 1808 } 1809 #if 0 1810 /* This doesn't work, because pmap_protect 1811 doesn't flush changes on pages that it 1812 has write-protected. */ 1813 1814 /* If the page is not writable and this 1815 is the source, then there is no need 1816 to flush it from the cache. */ 1817 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1818 continue; 1819 #endif 1820 if (cache_needs_cleaning){ 1821 page_to_clean = 0; 1822 break; 1823 } 1824 else 1825 page_to_clean = npv->pv_va; 1826 cache_needs_cleaning = 1; 1827 } 1828 } 1829 1830 if (page_to_clean) 1831 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1832 else if (cache_needs_cleaning) { 1833 cpu_idcache_wbinv_all(); 1834 return (1); 1835 } 1836 return (0); 1837 } 1838 1839 /* 1840 * pmap_find_pv() 1841 * 1842 * This is a local function that finds a PV head for a given physical page. 1843 * This is a common op, and this function removes loads of ifdefs in the code. 1844 */ 1845 static __inline struct pv_head * 1846 pmap_find_pvh(phys) 1847 paddr_t phys; 1848 { 1849 int bank, off; 1850 struct pv_head *pvh; 1851 1852 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) 1853 panic("pmap_find_pv: not a real page, phys=%lx\n", phys); 1854 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 1855 return (pvh); 1856 } 1857 1858 /* 1859 * pmap_zero_page() 1860 * 1861 * Zero a given physical page by mapping it at a page hook point. 1862 * In doing the zero page op, the page we zero is mapped cachable, as with 1863 * StrongARM accesses to non-cached pages are non-burst making writing 1864 * _any_ bulk data very slow. 1865 */ 1866 void 1867 pmap_zero_page(phys) 1868 paddr_t phys; 1869 { 1870 struct pv_head *pvh; 1871 1872 /* Get an entry for this page, and clean it it. */ 1873 pvh = pmap_find_pvh(phys); 1874 simple_lock(&pvh->pvh_lock); 1875 pmap_clean_page(pvh->pvh_list, FALSE); 1876 simple_unlock(&pvh->pvh_lock); 1877 1878 /* 1879 * Hook in the page, zero it, and purge the cache for that 1880 * zeroed page. Invalidate the TLB as needed. 1881 */ 1882 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1883 cpu_tlb_flushD_SE(page_hook0.va); 1884 cpu_cpwait(); 1885 bzero_page(page_hook0.va); 1886 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1887 } 1888 1889 /* pmap_pageidlezero() 1890 * 1891 * The same as above, except that we assume that the page is not 1892 * mapped. This means we never have to flush the cache first. Called 1893 * from the idle loop. 1894 */ 1895 boolean_t 1896 pmap_pageidlezero(phys) 1897 paddr_t phys; 1898 { 1899 int i, *ptr; 1900 boolean_t rv = TRUE; 1901 1902 #ifdef DIAGNOSTIC 1903 struct pv_head *pvh; 1904 1905 pvh = pmap_find_pvh(phys); 1906 if (pvh->pvh_list != NULL) 1907 panic("pmap_pageidlezero: zeroing mapped page\n"); 1908 #endif 1909 1910 /* 1911 * Hook in the page, zero it, and purge the cache for that 1912 * zeroed page. Invalidate the TLB as needed. 1913 */ 1914 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1915 cpu_tlb_flushD_SE(page_hook0.va); 1916 cpu_cpwait(); 1917 1918 for (i = 0, ptr = (int *)page_hook0.va; 1919 i < (NBPG / sizeof(int)); i++) { 1920 if (sched_whichqs != 0) { 1921 /* 1922 * A process has become ready. Abort now, 1923 * so we don't keep it waiting while we 1924 * do slow memory access to finish this 1925 * page. 1926 */ 1927 rv = FALSE; 1928 break; 1929 } 1930 *ptr++ = 0; 1931 } 1932 1933 if (rv) 1934 /* 1935 * if we aborted we'll rezero this page again later so don't 1936 * purge it unless we finished it 1937 */ 1938 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1939 return (rv); 1940 } 1941 1942 /* 1943 * pmap_copy_page() 1944 * 1945 * Copy one physical page into another, by mapping the pages into 1946 * hook points. The same comment regarding cachability as in 1947 * pmap_zero_page also applies here. 1948 */ 1949 void 1950 pmap_copy_page(src, dest) 1951 paddr_t src; 1952 paddr_t dest; 1953 { 1954 struct pv_head *src_pvh, *dest_pvh; 1955 boolean_t cleanedcache; 1956 1957 /* Get PV entries for the pages, and clean them if needed. */ 1958 src_pvh = pmap_find_pvh(src); 1959 1960 simple_lock(&src_pvh->pvh_lock); 1961 cleanedcache = pmap_clean_page(src_pvh->pvh_list, TRUE); 1962 simple_unlock(&src_pvh->pvh_lock); 1963 1964 if (cleanedcache == 0) { 1965 dest_pvh = pmap_find_pvh(dest); 1966 simple_lock(&dest_pvh->pvh_lock); 1967 pmap_clean_page(dest_pvh->pvh_list, FALSE); 1968 simple_unlock(&dest_pvh->pvh_lock); 1969 } 1970 /* 1971 * Map the pages into the page hook points, copy them, and purge 1972 * the cache for the appropriate page. Invalidate the TLB 1973 * as required. 1974 */ 1975 *page_hook0.pte = L2_PTE(src & PG_FRAME, AP_KRW); 1976 *page_hook1.pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1977 cpu_tlb_flushD_SE(page_hook0.va); 1978 cpu_tlb_flushD_SE(page_hook1.va); 1979 cpu_cpwait(); 1980 bcopy_page(page_hook0.va, page_hook1.va); 1981 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1982 cpu_dcache_wbinv_range(page_hook1.va, NBPG); 1983 } 1984 1985 #if 0 1986 void 1987 pmap_pte_addref(pmap, va) 1988 struct pmap *pmap; 1989 vaddr_t va; 1990 { 1991 pd_entry_t *pde; 1992 paddr_t pa; 1993 struct vm_page *m; 1994 1995 if (pmap == pmap_kernel()) 1996 return; 1997 1998 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1999 pa = pmap_pte_pa(pde); 2000 m = PHYS_TO_VM_PAGE(pa); 2001 ++m->wire_count; 2002 #ifdef MYCROFT_HACK 2003 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2004 pmap, va, pde, pa, m, m->wire_count); 2005 #endif 2006 } 2007 2008 void 2009 pmap_pte_delref(pmap, va) 2010 struct pmap *pmap; 2011 vaddr_t va; 2012 { 2013 pd_entry_t *pde; 2014 paddr_t pa; 2015 struct vm_page *m; 2016 2017 if (pmap == pmap_kernel()) 2018 return; 2019 2020 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 2021 pa = pmap_pte_pa(pde); 2022 m = PHYS_TO_VM_PAGE(pa); 2023 --m->wire_count; 2024 #ifdef MYCROFT_HACK 2025 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 2026 pmap, va, pde, pa, m, m->wire_count); 2027 #endif 2028 if (m->wire_count == 0) { 2029 #ifdef MYCROFT_HACK 2030 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 2031 pmap, va, pde, pa, m); 2032 #endif 2033 pmap_unmap_in_l1(pmap, va); 2034 uvm_pagefree(m); 2035 --pmap->pm_stats.resident_count; 2036 } 2037 } 2038 #else 2039 #define pmap_pte_addref(pmap, va) 2040 #define pmap_pte_delref(pmap, va) 2041 #endif 2042 2043 /* 2044 * Since we have a virtually indexed cache, we may need to inhibit caching if 2045 * there is more than one mapping and at least one of them is writable. 2046 * Since we purge the cache on every context switch, we only need to check for 2047 * other mappings within the same pmap, or kernel_pmap. 2048 * This function is also called when a page is unmapped, to possibly reenable 2049 * caching on any remaining mappings. 2050 * 2051 * The code implements the following logic, where: 2052 * 2053 * KW = # of kernel read/write pages 2054 * KR = # of kernel read only pages 2055 * UW = # of user read/write pages 2056 * UR = # of user read only pages 2057 * OW = # of user read/write pages in another pmap, then 2058 * 2059 * KC = kernel mapping is cacheable 2060 * UC = user mapping is cacheable 2061 * 2062 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2063 * +--------------------------------------------- 2064 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2065 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2066 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2067 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2068 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2069 * 2070 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2071 */ 2072 __inline static void 2073 pmap_vac_me_harder(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2074 boolean_t clear_cache) 2075 { 2076 if (pmap == pmap_kernel()) 2077 pmap_vac_me_kpmap(pmap, pvh, ptes, clear_cache); 2078 else 2079 pmap_vac_me_user(pmap, pvh, ptes, clear_cache); 2080 } 2081 2082 static void 2083 pmap_vac_me_kpmap(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2084 boolean_t clear_cache) 2085 { 2086 int user_entries = 0; 2087 int user_writable = 0; 2088 int user_cacheable = 0; 2089 int kernel_entries = 0; 2090 int kernel_writable = 0; 2091 int kernel_cacheable = 0; 2092 struct pv_entry *pv; 2093 struct pmap *last_pmap = pmap; 2094 2095 #ifdef DIAGNOSTIC 2096 if (pmap != pmap_kernel()) 2097 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2098 #endif 2099 2100 /* 2101 * Pass one, see if there are both kernel and user pmaps for 2102 * this page. Calculate whether there are user-writable or 2103 * kernel-writable pages. 2104 */ 2105 for (pv = pvh->pvh_list; pv != NULL; pv = pv->pv_next) { 2106 if (pv->pv_pmap != pmap) { 2107 user_entries++; 2108 if (pv->pv_flags & PT_Wr) 2109 user_writable++; 2110 if ((pv->pv_flags & PT_NC) == 0) 2111 user_cacheable++; 2112 } else { 2113 kernel_entries++; 2114 if (pv->pv_flags & PT_Wr) 2115 kernel_writable++; 2116 if ((pv->pv_flags & PT_NC) == 0) 2117 kernel_cacheable++; 2118 } 2119 } 2120 2121 /* 2122 * We know we have just been updating a kernel entry, so if 2123 * all user pages are already cacheable, then there is nothing 2124 * further to do. 2125 */ 2126 if (kernel_entries == 0 && 2127 user_cacheable == user_entries) 2128 return; 2129 2130 if (user_entries) { 2131 /* 2132 * Scan over the list again, for each entry, if it 2133 * might not be set correctly, call pmap_vac_me_user 2134 * to recalculate the settings. 2135 */ 2136 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) { 2137 /* 2138 * We know kernel mappings will get set 2139 * correctly in other calls. We also know 2140 * that if the pmap is the same as last_pmap 2141 * then we've just handled this entry. 2142 */ 2143 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2144 continue; 2145 /* 2146 * If there are kernel entries and this page 2147 * is writable but non-cacheable, then we can 2148 * skip this entry also. 2149 */ 2150 if (kernel_entries > 0 && 2151 (pv->pv_flags & (PT_NC | PT_Wr)) == 2152 (PT_NC | PT_Wr)) 2153 continue; 2154 /* 2155 * Similarly if there are no kernel-writable 2156 * entries and the page is already 2157 * read-only/cacheable. 2158 */ 2159 if (kernel_writable == 0 && 2160 (pv->pv_flags & (PT_NC | PT_Wr)) == 0) 2161 continue; 2162 /* 2163 * For some of the remaining cases, we know 2164 * that we must recalculate, but for others we 2165 * can't tell if they are correct or not, so 2166 * we recalculate anyway. 2167 */ 2168 pmap_unmap_ptes(last_pmap); 2169 last_pmap = pv->pv_pmap; 2170 ptes = pmap_map_ptes(last_pmap); 2171 pmap_vac_me_user(last_pmap, pvh, ptes, 2172 pmap_is_curpmap(last_pmap)); 2173 } 2174 /* Restore the pte mapping that was passed to us. */ 2175 if (last_pmap != pmap) { 2176 pmap_unmap_ptes(last_pmap); 2177 ptes = pmap_map_ptes(pmap); 2178 } 2179 if (kernel_entries == 0) 2180 return; 2181 } 2182 2183 pmap_vac_me_user(pmap, pvh, ptes, clear_cache); 2184 return; 2185 } 2186 2187 static void 2188 pmap_vac_me_user(struct pmap *pmap, struct pv_head *pvh, pt_entry_t *ptes, 2189 boolean_t clear_cache) 2190 { 2191 struct pmap *kpmap = pmap_kernel(); 2192 struct pv_entry *pv, *npv; 2193 int entries = 0; 2194 int writable = 0; 2195 int cacheable_entries = 0; 2196 int kern_cacheable = 0; 2197 int other_writable = 0; 2198 2199 pv = pvh->pvh_list; 2200 KASSERT(ptes != NULL); 2201 2202 /* 2203 * Count mappings and writable mappings in this pmap. 2204 * Include kernel mappings as part of our own. 2205 * Keep a pointer to the first one. 2206 */ 2207 for (npv = pv; npv; npv = npv->pv_next) { 2208 /* Count mappings in the same pmap */ 2209 if (pmap == npv->pv_pmap || 2210 kpmap == npv->pv_pmap) { 2211 if (entries++ == 0) 2212 pv = npv; 2213 /* Cacheable mappings */ 2214 if ((npv->pv_flags & PT_NC) == 0) { 2215 cacheable_entries++; 2216 if (kpmap == npv->pv_pmap) 2217 kern_cacheable++; 2218 } 2219 /* Writable mappings */ 2220 if (npv->pv_flags & PT_Wr) 2221 ++writable; 2222 } else if (npv->pv_flags & PT_Wr) 2223 other_writable = 1; 2224 } 2225 2226 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2227 "writable %d cacheable %d %s\n", pmap, entries, writable, 2228 cacheable_entries, clear_cache ? "clean" : "no clean")); 2229 2230 /* 2231 * Enable or disable caching as necessary. 2232 * Note: the first entry might be part of the kernel pmap, 2233 * so we can't assume this is indicative of the state of the 2234 * other (maybe non-kpmap) entries. 2235 */ 2236 if ((entries > 1 && writable) || 2237 (entries > 0 && pmap == kpmap && other_writable)) { 2238 if (cacheable_entries == 0) 2239 return; 2240 for (npv = pv; npv; npv = npv->pv_next) { 2241 if ((pmap == npv->pv_pmap 2242 || kpmap == npv->pv_pmap) && 2243 (npv->pv_flags & PT_NC) == 0) { 2244 ptes[arm_byte_to_page(npv->pv_va)] &= 2245 ~(PT_C | PT_B); 2246 npv->pv_flags |= PT_NC; 2247 /* 2248 * If this page needs flushing from the 2249 * cache, and we aren't going to do it 2250 * below, do it now. 2251 */ 2252 if ((cacheable_entries < 4 && 2253 (clear_cache || npv->pv_pmap == kpmap)) || 2254 (npv->pv_pmap == kpmap && 2255 !clear_cache && kern_cacheable < 4)) { 2256 cpu_idcache_wbinv_range(npv->pv_va, 2257 NBPG); 2258 cpu_tlb_flushID_SE(npv->pv_va); 2259 } 2260 } 2261 } 2262 if ((clear_cache && cacheable_entries >= 4) || 2263 kern_cacheable >= 4) { 2264 cpu_idcache_wbinv_all(); 2265 cpu_tlb_flushID(); 2266 } 2267 cpu_cpwait(); 2268 } else if (entries > 0) { 2269 /* 2270 * Turn cacheing back on for some pages. If it is a kernel 2271 * page, only do so if there are no other writable pages. 2272 */ 2273 for (npv = pv; npv; npv = npv->pv_next) { 2274 if ((pmap == npv->pv_pmap || 2275 (kpmap == npv->pv_pmap && other_writable == 0)) && 2276 (npv->pv_flags & PT_NC)) { 2277 ptes[arm_byte_to_page(npv->pv_va)] |= 2278 pte_cache_mode; 2279 npv->pv_flags &= ~PT_NC; 2280 } 2281 } 2282 } 2283 } 2284 2285 /* 2286 * pmap_remove() 2287 * 2288 * pmap_remove is responsible for nuking a number of mappings for a range 2289 * of virtual address space in the current pmap. To do this efficiently 2290 * is interesting, because in a number of cases a wide virtual address 2291 * range may be supplied that contains few actual mappings. So, the 2292 * optimisations are: 2293 * 1. Try and skip over hunks of address space for which an L1 entry 2294 * does not exist. 2295 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2296 * maybe do just a partial cache clean. This path of execution is 2297 * complicated by the fact that the cache must be flushed _before_ 2298 * the PTE is nuked, being a VAC :-) 2299 * 3. Maybe later fast-case a single page, but I don't think this is 2300 * going to make _that_ much difference overall. 2301 */ 2302 2303 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2304 2305 void 2306 pmap_remove(pmap, sva, eva) 2307 struct pmap *pmap; 2308 vaddr_t sva; 2309 vaddr_t eva; 2310 { 2311 int cleanlist_idx = 0; 2312 struct pagelist { 2313 vaddr_t va; 2314 pt_entry_t *pte; 2315 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2316 pt_entry_t *pte = 0, *ptes; 2317 paddr_t pa; 2318 int pmap_active; 2319 struct pv_head *pvh; 2320 2321 /* Exit quick if there is no pmap */ 2322 if (!pmap) 2323 return; 2324 2325 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2326 2327 sva &= PG_FRAME; 2328 eva &= PG_FRAME; 2329 2330 /* 2331 * we lock in the pmap => pv_head direction 2332 */ 2333 PMAP_MAP_TO_HEAD_LOCK(); 2334 2335 ptes = pmap_map_ptes(pmap); 2336 /* Get a page table pointer */ 2337 while (sva < eva) { 2338 if (pmap_pde_page(pmap_pde(pmap, sva))) 2339 break; 2340 sva = (sva & PD_MASK) + NBPD; 2341 } 2342 2343 pte = &ptes[arm_byte_to_page(sva)]; 2344 /* Note if the pmap is active thus require cache and tlb cleans */ 2345 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2346 || (pmap == pmap_kernel())) 2347 pmap_active = 1; 2348 else 2349 pmap_active = 0; 2350 2351 /* Now loop along */ 2352 while (sva < eva) { 2353 /* Check if we can move to the next PDE (l1 chunk) */ 2354 if (!(sva & PT_MASK)) 2355 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2356 sva += NBPD; 2357 pte += arm_byte_to_page(NBPD); 2358 continue; 2359 } 2360 2361 /* We've found a valid PTE, so this page of PTEs has to go. */ 2362 if (pmap_pte_v(pte)) { 2363 int bank, off; 2364 2365 /* Update statistics */ 2366 --pmap->pm_stats.resident_count; 2367 2368 /* 2369 * Add this page to our cache remove list, if we can. 2370 * If, however the cache remove list is totally full, 2371 * then do a complete cache invalidation taking note 2372 * to backtrack the PTE table beforehand, and ignore 2373 * the lists in future because there's no longer any 2374 * point in bothering with them (we've paid the 2375 * penalty, so will carry on unhindered). Otherwise, 2376 * when we fall out, we just clean the list. 2377 */ 2378 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2379 pa = pmap_pte_pa(pte); 2380 2381 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2382 /* Add to the clean list. */ 2383 cleanlist[cleanlist_idx].pte = pte; 2384 cleanlist[cleanlist_idx].va = sva; 2385 cleanlist_idx++; 2386 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2387 int cnt; 2388 2389 /* Nuke everything if needed. */ 2390 if (pmap_active) { 2391 cpu_idcache_wbinv_all(); 2392 cpu_tlb_flushID(); 2393 } 2394 2395 /* 2396 * Roll back the previous PTE list, 2397 * and zero out the current PTE. 2398 */ 2399 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2400 *cleanlist[cnt].pte = 0; 2401 pmap_pte_delref(pmap, cleanlist[cnt].va); 2402 } 2403 *pte = 0; 2404 pmap_pte_delref(pmap, sva); 2405 cleanlist_idx++; 2406 } else { 2407 /* 2408 * We've already nuked the cache and 2409 * TLB, so just carry on regardless, 2410 * and we won't need to do it again 2411 */ 2412 *pte = 0; 2413 pmap_pte_delref(pmap, sva); 2414 } 2415 2416 /* 2417 * Update flags. In a number of circumstances, 2418 * we could cluster a lot of these and do a 2419 * number of sequential pages in one go. 2420 */ 2421 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2422 struct pv_entry *pve; 2423 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2424 simple_lock(&pvh->pvh_lock); 2425 pve = pmap_remove_pv(pvh, pmap, sva); 2426 pmap_free_pv(pmap, pve); 2427 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2428 simple_unlock(&pvh->pvh_lock); 2429 } 2430 } 2431 sva += NBPG; 2432 pte++; 2433 } 2434 2435 pmap_unmap_ptes(pmap); 2436 /* 2437 * Now, if we've fallen through down to here, chances are that there 2438 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2439 */ 2440 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2441 u_int cnt; 2442 2443 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2444 if (pmap_active) { 2445 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2446 NBPG); 2447 *cleanlist[cnt].pte = 0; 2448 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2449 } else 2450 *cleanlist[cnt].pte = 0; 2451 pmap_pte_delref(pmap, cleanlist[cnt].va); 2452 } 2453 } 2454 PMAP_MAP_TO_HEAD_UNLOCK(); 2455 } 2456 2457 /* 2458 * Routine: pmap_remove_all 2459 * Function: 2460 * Removes this physical page from 2461 * all physical maps in which it resides. 2462 * Reflects back modify bits to the pager. 2463 */ 2464 2465 static void 2466 pmap_remove_all(pa) 2467 paddr_t pa; 2468 { 2469 struct pv_entry *pv, *npv; 2470 struct pv_head *pvh; 2471 struct pmap *pmap; 2472 pt_entry_t *pte, *ptes; 2473 2474 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", pa)); 2475 2476 /* set pv_head => pmap locking */ 2477 PMAP_HEAD_TO_MAP_LOCK(); 2478 2479 pvh = pmap_find_pvh(pa); 2480 simple_lock(&pvh->pvh_lock); 2481 2482 pv = pvh->pvh_list; 2483 if (pv == NULL) 2484 { 2485 PDEBUG(0, printf("free page\n")); 2486 simple_unlock(&pvh->pvh_lock); 2487 PMAP_HEAD_TO_MAP_UNLOCK(); 2488 return; 2489 } 2490 pmap_clean_page(pv, FALSE); 2491 2492 while (pv) { 2493 pmap = pv->pv_pmap; 2494 ptes = pmap_map_ptes(pmap); 2495 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2496 2497 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2498 pv->pv_va, pv->pv_flags)); 2499 #ifdef DEBUG 2500 if (!pmap_pde_page(pmap_pde(pmap, pv->pv_va)) || 2501 !pmap_pte_v(pte) || pmap_pte_pa(pte) != pa) 2502 panic("pmap_remove_all: bad mapping"); 2503 #endif /* DEBUG */ 2504 2505 /* 2506 * Update statistics 2507 */ 2508 --pmap->pm_stats.resident_count; 2509 2510 /* Wired bit */ 2511 if (pv->pv_flags & PT_W) 2512 --pmap->pm_stats.wired_count; 2513 2514 /* 2515 * Invalidate the PTEs. 2516 * XXX: should cluster them up and invalidate as many 2517 * as possible at once. 2518 */ 2519 2520 #ifdef needednotdone 2521 reduce wiring count on page table pages as references drop 2522 #endif 2523 2524 *pte = 0; 2525 pmap_pte_delref(pmap, pv->pv_va); 2526 2527 npv = pv->pv_next; 2528 pmap_free_pv(pmap, pv); 2529 pv = npv; 2530 pmap_unmap_ptes(pmap); 2531 } 2532 pvh->pvh_list = NULL; 2533 simple_unlock(&pvh->pvh_lock); 2534 PMAP_HEAD_TO_MAP_UNLOCK(); 2535 2536 PDEBUG(0, printf("done\n")); 2537 cpu_tlb_flushID(); 2538 cpu_cpwait(); 2539 } 2540 2541 2542 /* 2543 * Set the physical protection on the specified range of this map as requested. 2544 */ 2545 2546 void 2547 pmap_protect(pmap, sva, eva, prot) 2548 struct pmap *pmap; 2549 vaddr_t sva; 2550 vaddr_t eva; 2551 vm_prot_t prot; 2552 { 2553 pt_entry_t *pte = NULL, *ptes; 2554 int armprot; 2555 int flush = 0; 2556 paddr_t pa; 2557 int bank, off; 2558 struct pv_head *pvh; 2559 2560 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2561 pmap, sva, eva, prot)); 2562 2563 if (~prot & VM_PROT_READ) { 2564 /* Just remove the mappings. */ 2565 pmap_remove(pmap, sva, eva); 2566 /* pmap_update not needed as it should be called by the caller 2567 * of pmap_protect */ 2568 return; 2569 } 2570 if (prot & VM_PROT_WRITE) { 2571 /* 2572 * If this is a read->write transition, just ignore it and let 2573 * uvm_fault() take care of it later. 2574 */ 2575 return; 2576 } 2577 2578 sva &= PG_FRAME; 2579 eva &= PG_FRAME; 2580 2581 /* Need to lock map->head */ 2582 PMAP_MAP_TO_HEAD_LOCK(); 2583 2584 ptes = pmap_map_ptes(pmap); 2585 /* 2586 * We need to acquire a pointer to a page table page before entering 2587 * the following loop. 2588 */ 2589 while (sva < eva) { 2590 if (pmap_pde_page(pmap_pde(pmap, sva))) 2591 break; 2592 sva = (sva & PD_MASK) + NBPD; 2593 } 2594 2595 pte = &ptes[arm_byte_to_page(sva)]; 2596 2597 while (sva < eva) { 2598 /* only check once in a while */ 2599 if ((sva & PT_MASK) == 0) { 2600 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2601 /* We can race ahead here, to the next pde. */ 2602 sva += NBPD; 2603 pte += arm_byte_to_page(NBPD); 2604 continue; 2605 } 2606 } 2607 2608 if (!pmap_pte_v(pte)) 2609 goto next; 2610 2611 flush = 1; 2612 2613 armprot = 0; 2614 if (sva < VM_MAXUSER_ADDRESS) 2615 armprot |= PT_AP(AP_U); 2616 else if (sva < VM_MAX_ADDRESS) 2617 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2618 *pte = (*pte & 0xfffff00f) | armprot; 2619 2620 pa = pmap_pte_pa(pte); 2621 2622 /* Get the physical page index */ 2623 2624 /* Clear write flag */ 2625 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2626 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2627 simple_lock(&pvh->pvh_lock); 2628 (void) pmap_modify_pv(pmap, sva, pvh, PT_Wr, 0); 2629 pmap_vac_me_harder(pmap, pvh, ptes, FALSE); 2630 simple_unlock(&pvh->pvh_lock); 2631 } 2632 2633 next: 2634 sva += NBPG; 2635 pte++; 2636 } 2637 pmap_unmap_ptes(pmap); 2638 PMAP_MAP_TO_HEAD_UNLOCK(); 2639 if (flush) 2640 cpu_tlb_flushID(); 2641 } 2642 2643 /* 2644 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2645 * int flags) 2646 * 2647 * Insert the given physical page (p) at 2648 * the specified virtual address (v) in the 2649 * target physical map with the protection requested. 2650 * 2651 * If specified, the page will be wired down, meaning 2652 * that the related pte can not be reclaimed. 2653 * 2654 * NB: This is the only routine which MAY NOT lazy-evaluate 2655 * or lose information. That is, this routine must actually 2656 * insert this page into the given map NOW. 2657 */ 2658 2659 int 2660 pmap_enter(pmap, va, pa, prot, flags) 2661 struct pmap *pmap; 2662 vaddr_t va; 2663 paddr_t pa; 2664 vm_prot_t prot; 2665 int flags; 2666 { 2667 pt_entry_t *pte, *ptes; 2668 u_int npte; 2669 int bank, off; 2670 paddr_t opa; 2671 int nflags; 2672 boolean_t wired = (flags & PMAP_WIRED) != 0; 2673 struct pv_entry *pve; 2674 struct pv_head *pvh; 2675 int error; 2676 2677 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2678 va, pa, pmap, prot, wired)); 2679 2680 #ifdef DIAGNOSTIC 2681 /* Valid address ? */ 2682 if (va >= (KERNEL_VM_BASE + KERNEL_VM_SIZE)) 2683 panic("pmap_enter: too big"); 2684 if (pmap != pmap_kernel() && va != 0) { 2685 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2686 panic("pmap_enter: kernel page in user map"); 2687 } else { 2688 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2689 panic("pmap_enter: user page in kernel map"); 2690 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2691 panic("pmap_enter: entering PT page"); 2692 } 2693 #endif 2694 /* get lock */ 2695 PMAP_MAP_TO_HEAD_LOCK(); 2696 /* 2697 * Get a pointer to the pte for this virtual address. If the 2698 * pte pointer is NULL then we are missing the L2 page table 2699 * so we need to create one. 2700 */ 2701 /* XXX horrible hack to get us working with lockdebug */ 2702 simple_lock(&pmap->pm_obj.vmobjlock); 2703 pte = pmap_pte(pmap, va); 2704 if (!pte) { 2705 struct vm_page *ptp; 2706 2707 /* if failure is allowed then don't try too hard */ 2708 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2709 if (ptp == NULL) { 2710 if (flags & PMAP_CANFAIL) { 2711 error = ENOMEM; 2712 goto out; 2713 } 2714 panic("pmap_enter: get ptp failed"); 2715 } 2716 2717 pte = pmap_pte(pmap, va); 2718 #ifdef DIAGNOSTIC 2719 if (!pte) 2720 panic("pmap_enter: no pte"); 2721 #endif 2722 } 2723 2724 nflags = 0; 2725 if (prot & VM_PROT_WRITE) 2726 nflags |= PT_Wr; 2727 if (wired) 2728 nflags |= PT_W; 2729 2730 /* More debugging info */ 2731 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2732 *pte)); 2733 2734 /* Is the pte valid ? If so then this page is already mapped */ 2735 if (pmap_pte_v(pte)) { 2736 /* Get the physical address of the current page mapped */ 2737 opa = pmap_pte_pa(pte); 2738 2739 #ifdef MYCROFT_HACK 2740 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2741 #endif 2742 2743 /* Are we mapping the same page ? */ 2744 if (opa == pa) { 2745 /* All we must be doing is changing the protection */ 2746 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2747 va, pa)); 2748 2749 /* Has the wiring changed ? */ 2750 if ((bank = vm_physseg_find(atop(pa), &off)) != -1) { 2751 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2752 simple_lock(&pvh->pvh_lock); 2753 (void) pmap_modify_pv(pmap, va, pvh, 2754 PT_Wr | PT_W, nflags); 2755 simple_unlock(&pvh->pvh_lock); 2756 } else { 2757 pvh = NULL; 2758 } 2759 } else { 2760 /* We are replacing the page with a new one. */ 2761 cpu_idcache_wbinv_range(va, NBPG); 2762 2763 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2764 va, pa, opa)); 2765 2766 /* 2767 * If it is part of our managed memory then we 2768 * must remove it from the PV list 2769 */ 2770 if ((bank = vm_physseg_find(atop(opa), &off)) != -1) { 2771 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2772 simple_lock(&pvh->pvh_lock); 2773 pve = pmap_remove_pv(pvh, pmap, va); 2774 simple_unlock(&pvh->pvh_lock); 2775 } else { 2776 pve = NULL; 2777 } 2778 2779 goto enter; 2780 } 2781 } else { 2782 opa = 0; 2783 pve = NULL; 2784 pmap_pte_addref(pmap, va); 2785 2786 /* pte is not valid so we must be hooking in a new page */ 2787 ++pmap->pm_stats.resident_count; 2788 2789 enter: 2790 /* 2791 * Enter on the PV list if part of our managed memory 2792 */ 2793 bank = vm_physseg_find(atop(pa), &off); 2794 2795 if (pmap_initialized && (bank != -1)) { 2796 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 2797 if (pve == NULL) { 2798 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2799 if (pve == NULL) { 2800 if (flags & PMAP_CANFAIL) { 2801 error = ENOMEM; 2802 goto out; 2803 } 2804 panic("pmap_enter: no pv entries available"); 2805 } 2806 } 2807 /* enter_pv locks pvh when adding */ 2808 pmap_enter_pv(pvh, pve, pmap, va, NULL, nflags); 2809 } else { 2810 pvh = NULL; 2811 if (pve != NULL) 2812 pmap_free_pv(pmap, pve); 2813 } 2814 } 2815 2816 #ifdef MYCROFT_HACK 2817 if (mycroft_hack) 2818 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2819 #endif 2820 2821 /* Construct the pte, giving the correct access. */ 2822 npte = (pa & PG_FRAME); 2823 2824 /* VA 0 is magic. */ 2825 if (pmap != pmap_kernel() && va != 0) 2826 npte |= PT_AP(AP_U); 2827 2828 if (pmap_initialized && bank != -1) { 2829 #ifdef DIAGNOSTIC 2830 if ((flags & VM_PROT_ALL) & ~prot) 2831 panic("pmap_enter: access_type exceeds prot"); 2832 #endif 2833 npte |= pte_cache_mode; 2834 if (flags & VM_PROT_WRITE) { 2835 npte |= L2_SPAGE | PT_AP(AP_W); 2836 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 2837 } else if (flags & VM_PROT_ALL) { 2838 npte |= L2_SPAGE; 2839 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 2840 } else 2841 npte |= L2_INVAL; 2842 } else { 2843 if (prot & VM_PROT_WRITE) 2844 npte |= L2_SPAGE | PT_AP(AP_W); 2845 else if (prot & VM_PROT_ALL) 2846 npte |= L2_SPAGE; 2847 else 2848 npte |= L2_INVAL; 2849 } 2850 2851 #ifdef MYCROFT_HACK 2852 if (mycroft_hack) 2853 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2854 #endif 2855 2856 *pte = npte; 2857 2858 if (pmap_initialized && bank != -1) 2859 { 2860 boolean_t pmap_active = FALSE; 2861 /* XXX this will change once the whole of pmap_enter uses 2862 * map_ptes 2863 */ 2864 ptes = pmap_map_ptes(pmap); 2865 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2866 || (pmap == pmap_kernel())) 2867 pmap_active = TRUE; 2868 simple_lock(&pvh->pvh_lock); 2869 pmap_vac_me_harder(pmap, pvh, ptes, pmap_active); 2870 simple_unlock(&pvh->pvh_lock); 2871 pmap_unmap_ptes(pmap); 2872 } 2873 2874 /* Better flush the TLB ... */ 2875 cpu_tlb_flushID_SE(va); 2876 error = 0; 2877 out: 2878 simple_unlock(&pmap->pm_obj.vmobjlock); 2879 PMAP_MAP_TO_HEAD_UNLOCK(); 2880 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2881 2882 return error; 2883 } 2884 2885 void 2886 pmap_kenter_pa(va, pa, prot) 2887 vaddr_t va; 2888 paddr_t pa; 2889 vm_prot_t prot; 2890 { 2891 struct pmap *pmap = pmap_kernel(); 2892 pt_entry_t *pte; 2893 struct vm_page *pg; 2894 2895 if (!pmap_pde_page(pmap_pde(pmap, va))) { 2896 2897 #ifdef DIAGNOSTIC 2898 if (pmap_pde_v(pmap_pde(pmap, va))) 2899 panic("Trying to map kernel page into section mapping" 2900 " VA=%lx PA=%lx", va, pa); 2901 #endif 2902 /* 2903 * For the kernel pmaps it would be better to ensure 2904 * that they are always present, and to grow the 2905 * kernel as required. 2906 */ 2907 2908 /* must lock the pmap */ 2909 simple_lock(&(pmap_kernel()->pm_obj.vmobjlock)); 2910 /* Allocate a page table */ 2911 pg = uvm_pagealloc(&(pmap_kernel()->pm_obj), 0, NULL, 2912 UVM_PGA_USERESERVE | UVM_PGA_ZERO); 2913 if (pg == NULL) { 2914 panic("pmap_kenter_pa: no free pages"); 2915 } 2916 pg->flags &= ~PG_BUSY; /* never busy */ 2917 2918 /* Wire this page table into the L1. */ 2919 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(pg), TRUE); 2920 simple_unlock(&(pmap_kernel()->pm_obj.vmobjlock)); 2921 } 2922 pte = vtopte(va); 2923 KASSERT(!pmap_pte_v(pte)); 2924 *pte = L2_PTE(pa, AP_KRW); 2925 } 2926 2927 void 2928 pmap_kremove(va, len) 2929 vaddr_t va; 2930 vsize_t len; 2931 { 2932 pt_entry_t *pte; 2933 2934 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2935 2936 /* 2937 * We assume that we will only be called with small 2938 * regions of memory. 2939 */ 2940 2941 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2942 pte = vtopte(va); 2943 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2944 *pte = 0; 2945 cpu_tlb_flushID_SE(va); 2946 } 2947 } 2948 2949 /* 2950 * pmap_page_protect: 2951 * 2952 * Lower the permission for all mappings to a given page. 2953 */ 2954 2955 void 2956 pmap_page_protect(pg, prot) 2957 struct vm_page *pg; 2958 vm_prot_t prot; 2959 { 2960 paddr_t pa = VM_PAGE_TO_PHYS(pg); 2961 2962 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", pa, prot)); 2963 2964 switch(prot) { 2965 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2966 case VM_PROT_READ|VM_PROT_WRITE: 2967 return; 2968 2969 case VM_PROT_READ: 2970 case VM_PROT_READ|VM_PROT_EXECUTE: 2971 pmap_copy_on_write(pa); 2972 break; 2973 2974 default: 2975 pmap_remove_all(pa); 2976 break; 2977 } 2978 } 2979 2980 2981 /* 2982 * Routine: pmap_unwire 2983 * Function: Clear the wired attribute for a map/virtual-address 2984 * pair. 2985 * In/out conditions: 2986 * The mapping must already exist in the pmap. 2987 */ 2988 2989 void 2990 pmap_unwire(pmap, va) 2991 struct pmap *pmap; 2992 vaddr_t va; 2993 { 2994 pt_entry_t *pte; 2995 paddr_t pa; 2996 int bank, off; 2997 struct pv_head *pvh; 2998 2999 /* 3000 * Make sure pmap is valid. -dct 3001 */ 3002 if (pmap == NULL) 3003 return; 3004 3005 /* Get the pte */ 3006 pte = pmap_pte(pmap, va); 3007 if (!pte) 3008 return; 3009 3010 /* Extract the physical address of the page */ 3011 pa = pmap_pte_pa(pte); 3012 3013 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3014 return; 3015 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3016 simple_lock(&pvh->pvh_lock); 3017 /* Update the wired bit in the pv entry for this page. */ 3018 (void) pmap_modify_pv(pmap, va, pvh, PT_W, 0); 3019 simple_unlock(&pvh->pvh_lock); 3020 } 3021 3022 /* 3023 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 3024 * 3025 * Return the pointer to a page table entry corresponding to the supplied 3026 * virtual address. 3027 * 3028 * The page directory is first checked to make sure that a page table 3029 * for the address in question exists and if it does a pointer to the 3030 * entry is returned. 3031 * 3032 * The way this works is that that the kernel page tables are mapped 3033 * into the memory map at ALT_PAGE_TBLS_BASE to ALT_PAGE_TBLS_BASE+4MB. 3034 * This allows page tables to be located quickly. 3035 */ 3036 pt_entry_t * 3037 pmap_pte(pmap, va) 3038 struct pmap *pmap; 3039 vaddr_t va; 3040 { 3041 pt_entry_t *ptp; 3042 pt_entry_t *result; 3043 3044 /* The pmap must be valid */ 3045 if (!pmap) 3046 return(NULL); 3047 3048 /* Return the address of the pte */ 3049 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 3050 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 3051 3052 /* Do we have a valid pde ? If not we don't have a page table */ 3053 if (!pmap_pde_page(pmap_pde(pmap, va))) { 3054 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 3055 pmap_pde(pmap, va))); 3056 return(NULL); 3057 } 3058 3059 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 3060 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3061 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3062 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME))); 3063 3064 /* 3065 * If the pmap is the kernel pmap or the pmap is the active one 3066 * then we can just return a pointer to entry relative to 3067 * PROCESS_PAGE_TBLS_BASE. 3068 * Otherwise we need to map the page tables to an alternative 3069 * address and reference them there. 3070 */ 3071 if (pmap == pmap_kernel() || pmap->pm_pptpt 3072 == (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3073 + ((PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) & 3074 ~3) + (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 3075 ptp = (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3076 } else { 3077 struct proc *p = curproc; 3078 3079 /* If we don't have a valid curproc use proc0 */ 3080 /* Perhaps we should just use kernel_pmap instead */ 3081 if (p == NULL) 3082 p = &proc0; 3083 #ifdef DIAGNOSTIC 3084 /* 3085 * The pmap should always be valid for the process so 3086 * panic if it is not. 3087 */ 3088 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 3089 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 3090 va, p, p->p_vmspace); 3091 console_debugger(); 3092 } 3093 /* 3094 * The pmap for the current process should be mapped. If it 3095 * is not then we have a problem. 3096 */ 3097 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 3098 (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3099 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3100 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 3101 printf("pmap pagetable = P%08lx current = P%08x ", 3102 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3103 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3104 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & 3105 PG_FRAME)); 3106 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 3107 panic("pmap_pte: current and pmap mismatch\n"); 3108 } 3109 #endif 3110 3111 ptp = (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3112 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3113 pmap->pm_pptpt, FALSE); 3114 cpu_tlb_flushD(); 3115 cpu_cpwait(); 3116 } 3117 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 3118 ((va >> (PGSHIFT-2)) & ~3))); 3119 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 3120 return(result); 3121 } 3122 3123 /* 3124 * Routine: pmap_extract 3125 * Function: 3126 * Extract the physical page address associated 3127 * with the given map/virtual_address pair. 3128 */ 3129 boolean_t 3130 pmap_extract(pmap, va, pap) 3131 struct pmap *pmap; 3132 vaddr_t va; 3133 paddr_t *pap; 3134 { 3135 pd_entry_t *pde; 3136 pt_entry_t *pte, *ptes; 3137 paddr_t pa; 3138 boolean_t rv = TRUE; 3139 3140 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 3141 3142 /* 3143 * Get the pte for this virtual address. 3144 */ 3145 pde = pmap_pde(pmap, va); 3146 ptes = pmap_map_ptes(pmap); 3147 pte = &ptes[arm_byte_to_page(va)]; 3148 3149 if (pmap_pde_section(pde)) { 3150 pa = (*pde & PD_MASK) | (va & (L1_SEC_SIZE - 1)); 3151 goto out; 3152 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3153 rv = FALSE; 3154 goto out; 3155 } 3156 3157 if ((*pte & L2_MASK) == L2_LPAGE) { 3158 /* Extract the physical address from the pte */ 3159 pa = *pte & ~(L2_LPAGE_SIZE - 1); 3160 3161 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3162 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3163 3164 if (pap != NULL) 3165 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3166 goto out; 3167 } 3168 3169 /* Extract the physical address from the pte */ 3170 pa = pmap_pte_pa(pte); 3171 3172 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3173 (pa | (va & ~PG_FRAME)))); 3174 3175 if (pap != NULL) 3176 *pap = pa | (va & ~PG_FRAME); 3177 out: 3178 pmap_unmap_ptes(pmap); 3179 return (rv); 3180 } 3181 3182 3183 /* 3184 * Copy the range specified by src_addr/len from the source map to the 3185 * range dst_addr/len in the destination map. 3186 * 3187 * This routine is only advisory and need not do anything. 3188 */ 3189 3190 void 3191 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3192 struct pmap *dst_pmap; 3193 struct pmap *src_pmap; 3194 vaddr_t dst_addr; 3195 vsize_t len; 3196 vaddr_t src_addr; 3197 { 3198 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3199 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3200 } 3201 3202 #if defined(PMAP_DEBUG) 3203 void 3204 pmap_dump_pvlist(phys, m) 3205 vaddr_t phys; 3206 char *m; 3207 { 3208 struct pv_head *pvh; 3209 struct pv_entry *pv; 3210 int bank, off; 3211 3212 if ((bank = vm_physseg_find(atop(phys), &off)) == -1) { 3213 printf("INVALID PA\n"); 3214 return; 3215 } 3216 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3217 simple_lock(&pvh->pvh_lock); 3218 printf("%s %08lx:", m, phys); 3219 if (pvh->pvh_list == NULL) { 3220 printf(" no mappings\n"); 3221 return; 3222 } 3223 3224 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) 3225 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3226 pv->pv_va, pv->pv_flags); 3227 3228 printf("\n"); 3229 simple_unlock(&pvh->pvh_lock); 3230 } 3231 3232 #endif /* PMAP_DEBUG */ 3233 3234 __inline static boolean_t 3235 pmap_testbit(pa, setbits) 3236 paddr_t pa; 3237 unsigned int setbits; 3238 { 3239 int bank, off; 3240 3241 PDEBUG(1, printf("pmap_testbit: pa=%08lx set=%08x\n", pa, setbits)); 3242 3243 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3244 return(FALSE); 3245 3246 /* 3247 * Check saved info only 3248 */ 3249 if (vm_physmem[bank].pmseg.attrs[off] & setbits) { 3250 PDEBUG(0, printf("pmap_attributes = %02x\n", 3251 vm_physmem[bank].pmseg.attrs[off])); 3252 return(TRUE); 3253 } 3254 3255 return(FALSE); 3256 } 3257 3258 static pt_entry_t * 3259 pmap_map_ptes(struct pmap *pmap) 3260 { 3261 struct proc *p; 3262 3263 /* the kernel's pmap is always accessible */ 3264 if (pmap == pmap_kernel()) { 3265 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE ; 3266 } 3267 3268 if (pmap_is_curpmap(pmap)) { 3269 simple_lock(&pmap->pm_obj.vmobjlock); 3270 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3271 } 3272 3273 p = curproc; 3274 3275 if (p == NULL) 3276 p = &proc0; 3277 3278 /* need to lock both curpmap and pmap: use ordered locking */ 3279 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3280 simple_lock(&pmap->pm_obj.vmobjlock); 3281 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3282 } else { 3283 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3284 simple_lock(&pmap->pm_obj.vmobjlock); 3285 } 3286 3287 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3288 pmap->pm_pptpt, FALSE); 3289 cpu_tlb_flushD(); 3290 cpu_cpwait(); 3291 return (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3292 } 3293 3294 /* 3295 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3296 */ 3297 3298 static void 3299 pmap_unmap_ptes(pmap) 3300 struct pmap *pmap; 3301 { 3302 if (pmap == pmap_kernel()) { 3303 return; 3304 } 3305 if (pmap_is_curpmap(pmap)) { 3306 simple_unlock(&pmap->pm_obj.vmobjlock); 3307 } else { 3308 simple_unlock(&pmap->pm_obj.vmobjlock); 3309 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3310 } 3311 } 3312 3313 /* 3314 * Modify pte bits for all ptes corresponding to the given physical address. 3315 * We use `maskbits' rather than `clearbits' because we're always passing 3316 * constants and the latter would require an extra inversion at run-time. 3317 */ 3318 3319 static void 3320 pmap_clearbit(pa, maskbits) 3321 paddr_t pa; 3322 unsigned int maskbits; 3323 { 3324 struct pv_entry *pv; 3325 struct pv_head *pvh; 3326 pt_entry_t *pte; 3327 vaddr_t va; 3328 int bank, off, tlbentry; 3329 3330 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3331 pa, maskbits)); 3332 3333 tlbentry = 0; 3334 3335 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3336 return; 3337 PMAP_HEAD_TO_MAP_LOCK(); 3338 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3339 simple_lock(&pvh->pvh_lock); 3340 3341 /* 3342 * Clear saved attributes (modify, reference) 3343 */ 3344 vm_physmem[bank].pmseg.attrs[off] &= ~maskbits; 3345 3346 if (pvh->pvh_list == NULL) { 3347 simple_unlock(&pvh->pvh_lock); 3348 PMAP_HEAD_TO_MAP_UNLOCK(); 3349 return; 3350 } 3351 3352 /* 3353 * Loop over all current mappings setting/clearing as appropos 3354 */ 3355 for (pv = pvh->pvh_list; pv; pv = pv->pv_next) { 3356 va = pv->pv_va; 3357 pv->pv_flags &= ~maskbits; 3358 pte = pmap_pte(pv->pv_pmap, va); 3359 KASSERT(pte != NULL); 3360 if (maskbits & (PT_Wr|PT_M)) { 3361 if ((pv->pv_flags & PT_NC)) { 3362 /* 3363 * Entry is not cacheable: reenable 3364 * the cache, nothing to flush 3365 * 3366 * Don't turn caching on again if this 3367 * is a modified emulation. This 3368 * would be inconsitent with the 3369 * settings created by 3370 * pmap_vac_me_harder(). 3371 * 3372 * There's no need to call 3373 * pmap_vac_me_harder() here: all 3374 * pages are loosing their write 3375 * permission. 3376 * 3377 */ 3378 if (maskbits & PT_Wr) { 3379 *pte |= pte_cache_mode; 3380 pv->pv_flags &= ~PT_NC; 3381 } 3382 } else if (pmap_is_curpmap(pv->pv_pmap)) 3383 /* 3384 * Entry is cacheable: check if pmap is 3385 * current if it is flush it, 3386 * otherwise it won't be in the cache 3387 */ 3388 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3389 3390 /* make the pte read only */ 3391 *pte &= ~PT_AP(AP_W); 3392 } 3393 3394 if (maskbits & PT_H) 3395 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3396 3397 if (pmap_is_curpmap(pv->pv_pmap)) 3398 /* 3399 * if we had cacheable pte's we'd clean the 3400 * pte out to memory here 3401 * 3402 * flush tlb entry as it's in the current pmap 3403 */ 3404 cpu_tlb_flushID_SE(pv->pv_va); 3405 } 3406 cpu_cpwait(); 3407 3408 simple_unlock(&pvh->pvh_lock); 3409 PMAP_HEAD_TO_MAP_UNLOCK(); 3410 } 3411 3412 3413 boolean_t 3414 pmap_clear_modify(pg) 3415 struct vm_page *pg; 3416 { 3417 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3418 boolean_t rv; 3419 3420 PDEBUG(0, printf("pmap_clear_modify pa=%08lx\n", pa)); 3421 rv = pmap_testbit(pa, PT_M); 3422 pmap_clearbit(pa, PT_M); 3423 return rv; 3424 } 3425 3426 3427 boolean_t 3428 pmap_clear_reference(pg) 3429 struct vm_page *pg; 3430 { 3431 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3432 boolean_t rv; 3433 3434 PDEBUG(0, printf("pmap_clear_reference pa=%08lx\n", pa)); 3435 rv = pmap_testbit(pa, PT_H); 3436 pmap_clearbit(pa, PT_H); 3437 return rv; 3438 } 3439 3440 3441 void 3442 pmap_copy_on_write(pa) 3443 paddr_t pa; 3444 { 3445 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", pa)); 3446 pmap_clearbit(pa, PT_Wr); 3447 } 3448 3449 3450 boolean_t 3451 pmap_is_modified(pg) 3452 struct vm_page *pg; 3453 { 3454 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3455 boolean_t result; 3456 3457 result = pmap_testbit(pa, PT_M); 3458 PDEBUG(1, printf("pmap_is_modified pa=%08lx %x\n", pa, result)); 3459 return (result); 3460 } 3461 3462 3463 boolean_t 3464 pmap_is_referenced(pg) 3465 struct vm_page *pg; 3466 { 3467 paddr_t pa = VM_PAGE_TO_PHYS(pg); 3468 boolean_t result; 3469 3470 result = pmap_testbit(pa, PT_H); 3471 PDEBUG(0, printf("pmap_is_referenced pa=%08lx %x\n", pa, result)); 3472 return (result); 3473 } 3474 3475 3476 int 3477 pmap_modified_emulation(pmap, va) 3478 struct pmap *pmap; 3479 vaddr_t va; 3480 { 3481 pt_entry_t *pte; 3482 paddr_t pa; 3483 int bank, off; 3484 struct pv_head *pvh; 3485 u_int flags; 3486 3487 PDEBUG(2, printf("pmap_modified_emulation\n")); 3488 3489 /* Get the pte */ 3490 pte = pmap_pte(pmap, va); 3491 if (!pte) { 3492 PDEBUG(2, printf("no pte\n")); 3493 return(0); 3494 } 3495 3496 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3497 3498 /* Check for a zero pte */ 3499 if (*pte == 0) 3500 return(0); 3501 3502 /* This can happen if user code tries to access kernel memory. */ 3503 if ((*pte & PT_AP(AP_W)) != 0) 3504 return (0); 3505 3506 /* Extract the physical address of the page */ 3507 pa = pmap_pte_pa(pte); 3508 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3509 return(0); 3510 3511 PMAP_HEAD_TO_MAP_LOCK(); 3512 /* Get the current flags for this page. */ 3513 pvh = &vm_physmem[bank].pmseg.pvhead[off]; 3514 /* XXX: needed if we hold head->map lock? */ 3515 simple_lock(&pvh->pvh_lock); 3516 3517 flags = pmap_modify_pv(pmap, va, pvh, 0, 0); 3518 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3519 3520 /* 3521 * Do the flags say this page is writable ? If not then it is a 3522 * genuine write fault. If yes then the write fault is our fault 3523 * as we did not reflect the write access in the PTE. Now we know 3524 * a write has occurred we can correct this and also set the 3525 * modified bit 3526 */ 3527 if (~flags & PT_Wr) { 3528 simple_unlock(&pvh->pvh_lock); 3529 PMAP_HEAD_TO_MAP_UNLOCK(); 3530 return(0); 3531 } 3532 3533 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3534 va, pte, *pte)); 3535 vm_physmem[bank].pmseg.attrs[off] |= PT_H | PT_M; 3536 3537 /* 3538 * Re-enable write permissions for the page. No need to call 3539 * pmap_vac_me_harder(), since this is just a 3540 * modified-emulation fault, and the PT_Wr bit isn't changing. We've 3541 * already set the cacheable bits based on the assumption that we 3542 * can write to this page. 3543 */ 3544 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3545 PDEBUG(0, printf("->(%08x)\n", *pte)); 3546 3547 simple_unlock(&pvh->pvh_lock); 3548 PMAP_HEAD_TO_MAP_UNLOCK(); 3549 /* Return, indicating the problem has been dealt with */ 3550 cpu_tlb_flushID_SE(va); 3551 cpu_cpwait(); 3552 return(1); 3553 } 3554 3555 3556 int 3557 pmap_handled_emulation(pmap, va) 3558 struct pmap *pmap; 3559 vaddr_t va; 3560 { 3561 pt_entry_t *pte; 3562 paddr_t pa; 3563 int bank, off; 3564 3565 PDEBUG(2, printf("pmap_handled_emulation\n")); 3566 3567 /* Get the pte */ 3568 pte = pmap_pte(pmap, va); 3569 if (!pte) { 3570 PDEBUG(2, printf("no pte\n")); 3571 return(0); 3572 } 3573 3574 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3575 3576 /* Check for a zero pte */ 3577 if (*pte == 0) 3578 return(0); 3579 3580 /* This can happen if user code tries to access kernel memory. */ 3581 if ((*pte & L2_MASK) != L2_INVAL) 3582 return (0); 3583 3584 /* Extract the physical address of the page */ 3585 pa = pmap_pte_pa(pte); 3586 if ((bank = vm_physseg_find(atop(pa), &off)) == -1) 3587 return(0); 3588 3589 /* 3590 * Ok we just enable the pte and mark the attibs as handled 3591 */ 3592 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3593 va, pte, *pte)); 3594 vm_physmem[bank].pmseg.attrs[off] |= PT_H; 3595 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3596 PDEBUG(0, printf("->(%08x)\n", *pte)); 3597 3598 /* Return, indicating the problem has been dealt with */ 3599 cpu_tlb_flushID_SE(va); 3600 cpu_cpwait(); 3601 return(1); 3602 } 3603 3604 3605 3606 3607 /* 3608 * pmap_collect: free resources held by a pmap 3609 * 3610 * => optional function. 3611 * => called when a process is swapped out to free memory. 3612 */ 3613 3614 void 3615 pmap_collect(pmap) 3616 struct pmap *pmap; 3617 { 3618 } 3619 3620 /* 3621 * Routine: pmap_procwr 3622 * 3623 * Function: 3624 * Synchronize caches corresponding to [addr, addr+len) in p. 3625 * 3626 */ 3627 void 3628 pmap_procwr(p, va, len) 3629 struct proc *p; 3630 vaddr_t va; 3631 int len; 3632 { 3633 /* We only need to do anything if it is the current process. */ 3634 if (p == curproc) 3635 cpu_icache_sync_range(va, len); 3636 } 3637 /* 3638 * PTP functions 3639 */ 3640 3641 /* 3642 * pmap_steal_ptp: Steal a PTP from somewhere else. 3643 * 3644 * This is just a placeholder, for now we never steal. 3645 */ 3646 3647 static struct vm_page * 3648 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3649 { 3650 return (NULL); 3651 } 3652 3653 /* 3654 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3655 * 3656 * => pmap should NOT be pmap_kernel() 3657 * => pmap should be locked 3658 */ 3659 3660 static struct vm_page * 3661 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3662 { 3663 struct vm_page *ptp; 3664 3665 if (pmap_pde_page(pmap_pde(pmap, va))) { 3666 3667 /* valid... check hint (saves us a PA->PG lookup) */ 3668 #if 0 3669 if (pmap->pm_ptphint && 3670 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3671 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3672 return (pmap->pm_ptphint); 3673 #endif 3674 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3675 #ifdef DIAGNOSTIC 3676 if (ptp == NULL) 3677 panic("pmap_get_ptp: unmanaged user PTP"); 3678 #endif 3679 // pmap->pm_ptphint = ptp; 3680 return(ptp); 3681 } 3682 3683 /* allocate a new PTP (updates ptphint) */ 3684 return(pmap_alloc_ptp(pmap, va, just_try)); 3685 } 3686 3687 /* 3688 * pmap_alloc_ptp: allocate a PTP for a PMAP 3689 * 3690 * => pmap should already be locked by caller 3691 * => we use the ptp's wire_count to count the number of active mappings 3692 * in the PTP (we start it at one to prevent any chance this PTP 3693 * will ever leak onto the active/inactive queues) 3694 */ 3695 3696 /*__inline */ static struct vm_page * 3697 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3698 { 3699 struct vm_page *ptp; 3700 3701 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3702 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3703 if (ptp == NULL) { 3704 if (just_try) 3705 return (NULL); 3706 3707 ptp = pmap_steal_ptp(pmap, va); 3708 3709 if (ptp == NULL) 3710 return (NULL); 3711 /* Stole a page, zero it. */ 3712 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3713 } 3714 3715 /* got one! */ 3716 ptp->flags &= ~PG_BUSY; /* never busy */ 3717 ptp->wire_count = 1; /* no mappings yet */ 3718 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3719 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3720 // pmap->pm_ptphint = ptp; 3721 return (ptp); 3722 } 3723 3724 /* End of pmap.c */ 3725
Indexes created Thu Oct 23 22:10:10 GMT 2025