pmap.c revision 1.49
1 /* $NetBSD: pmap.c,v 1.49 2002/03/05 04:19:59 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.49 2002/03/05 04:19:59 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static struct pmap_head pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 pagehook_t page_hook0; 199 pagehook_t page_hook1; 200 char *memhook; 201 pt_entry_t msgbufpte; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 static void pmap_free_l1pt __P((struct l1pt *)); 271 static int pmap_allocpagedir __P((struct pmap *)); 272 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 273 static void pmap_remove_all __P((struct vm_page *)); 274 275 276 vsize_t npages; 277 278 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 279 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 280 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 281 __inline static boolean_t pmap_testbit __P((struct vm_page *, unsigned int)); 282 283 extern paddr_t physical_start; 284 extern paddr_t physical_freestart; 285 extern paddr_t physical_end; 286 extern paddr_t physical_freeend; 287 extern unsigned int free_pages; 288 extern int max_processes; 289 290 vaddr_t virtual_start; 291 vaddr_t virtual_end; 292 vaddr_t pmap_curmaxkvaddr; 293 294 vaddr_t avail_start; 295 vaddr_t avail_end; 296 297 extern pv_addr_t systempage; 298 299 #define ALLOC_PAGE_HOOK(x, s) \ 300 x.va = virtual_start; \ 301 x.pte = (pt_entry_t *)pmap_pte(pmap_kernel(), virtual_start); \ 302 virtual_start += s; 303 304 /* Variables used by the L1 page table queue code */ 305 SIMPLEQ_HEAD(l1pt_queue, l1pt); 306 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 307 int l1pt_static_queue_count; /* items in the static l1 queue */ 308 int l1pt_static_create_count; /* static l1 items created */ 309 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 310 int l1pt_queue_count; /* items in the l1 queue */ 311 int l1pt_create_count; /* stat - L1's create count */ 312 int l1pt_reuse_count; /* stat - L1's reused count */ 313 314 /* Local function prototypes (not used outside this file) */ 315 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 316 void pmap_copy_on_write __P((struct vm_page *)); 317 void pmap_pinit __P((struct pmap *)); 318 void pmap_freepagedir __P((struct pmap *)); 319 320 /* Other function prototypes */ 321 extern void bzero_page __P((vaddr_t)); 322 extern void bcopy_page __P((vaddr_t, vaddr_t)); 323 324 struct l1pt *pmap_alloc_l1pt __P((void)); 325 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 326 vaddr_t l2pa, boolean_t)); 327 328 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 329 static void pmap_unmap_ptes __P((struct pmap *)); 330 331 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 332 pt_entry_t *, boolean_t)); 333 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 334 pt_entry_t *, boolean_t)); 335 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 336 pt_entry_t *, boolean_t)); 337 338 /* 339 * Cache enable bits in PTE to use on pages that are cacheable. 340 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 341 * can chose between write-through and write-back cacheing. 342 */ 343 pt_entry_t pte_cache_mode = (PT_C | PT_B); 344 345 /* 346 * real definition of pv_entry. 347 */ 348 349 struct pv_entry { 350 struct pv_entry *pv_next; /* next pv_entry */ 351 struct pmap *pv_pmap; /* pmap where mapping lies */ 352 vaddr_t pv_va; /* virtual address for mapping */ 353 int pv_flags; /* flags */ 354 struct vm_page *pv_ptp; /* vm_page for the ptp */ 355 }; 356 357 /* 358 * pv_entrys are dynamically allocated in chunks from a single page. 359 * we keep track of how many pv_entrys are in use for each page and 360 * we can free pv_entry pages if needed. there is one lock for the 361 * entire allocation system. 362 */ 363 364 struct pv_page_info { 365 TAILQ_ENTRY(pv_page) pvpi_list; 366 struct pv_entry *pvpi_pvfree; 367 int pvpi_nfree; 368 }; 369 370 /* 371 * number of pv_entry's in a pv_page 372 * (note: won't work on systems where NPBG isn't a constant) 373 */ 374 375 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 376 sizeof(struct pv_entry)) 377 378 /* 379 * a pv_page: where pv_entrys are allocated from 380 */ 381 382 struct pv_page { 383 struct pv_page_info pvinfo; 384 struct pv_entry pvents[PVE_PER_PVPAGE]; 385 }; 386 387 #ifdef MYCROFT_HACK 388 int mycroft_hack = 0; 389 #endif 390 391 /* Function to set the debug level of the pmap code */ 392 393 #ifdef PMAP_DEBUG 394 void 395 pmap_debug(level) 396 int level; 397 { 398 pmap_debug_level = level; 399 printf("pmap_debug: level=%d\n", pmap_debug_level); 400 } 401 #endif /* PMAP_DEBUG */ 402 403 __inline static boolean_t 404 pmap_is_curpmap(struct pmap *pmap) 405 { 406 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 407 || (pmap == pmap_kernel())) 408 return (TRUE); 409 return (FALSE); 410 } 411 #include "isadma.h" 412 413 #if NISADMA > 0 414 /* 415 * Used to protect memory for ISA DMA bounce buffers. If, when loading 416 * pages into the system, memory intersects with any of these ranges, 417 * the intersecting memory will be loaded into a lower-priority free list. 418 */ 419 bus_dma_segment_t *pmap_isa_dma_ranges; 420 int pmap_isa_dma_nranges; 421 422 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 423 paddr_t *, psize_t *)); 424 425 /* 426 * Check if a memory range intersects with an ISA DMA range, and 427 * return the page-rounded intersection if it does. The intersection 428 * will be placed on a lower-priority free list. 429 */ 430 boolean_t 431 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 432 paddr_t pa; 433 psize_t size; 434 paddr_t *pap; 435 psize_t *sizep; 436 { 437 bus_dma_segment_t *ds; 438 int i; 439 440 if (pmap_isa_dma_ranges == NULL) 441 return (FALSE); 442 443 for (i = 0, ds = pmap_isa_dma_ranges; 444 i < pmap_isa_dma_nranges; i++, ds++) { 445 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 446 /* 447 * Beginning of region intersects with this range. 448 */ 449 *pap = trunc_page(pa); 450 *sizep = round_page(min(pa + size, 451 ds->ds_addr + ds->ds_len) - pa); 452 return (TRUE); 453 } 454 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 455 /* 456 * End of region intersects with this range. 457 */ 458 *pap = trunc_page(ds->ds_addr); 459 *sizep = round_page(min((pa + size) - ds->ds_addr, 460 ds->ds_len)); 461 return (TRUE); 462 } 463 } 464 465 /* 466 * No intersection found. 467 */ 468 return (FALSE); 469 } 470 #endif /* NISADMA > 0 */ 471 472 /* 473 * p v _ e n t r y f u n c t i o n s 474 */ 475 476 /* 477 * pv_entry allocation functions: 478 * the main pv_entry allocation functions are: 479 * pmap_alloc_pv: allocate a pv_entry structure 480 * pmap_free_pv: free one pv_entry 481 * pmap_free_pvs: free a list of pv_entrys 482 * 483 * the rest are helper functions 484 */ 485 486 /* 487 * pmap_alloc_pv: inline function to allocate a pv_entry structure 488 * => we lock pvalloc_lock 489 * => if we fail, we call out to pmap_alloc_pvpage 490 * => 3 modes: 491 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 492 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 493 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 494 * one now 495 * 496 * "try" is for optional functions like pmap_copy(). 497 */ 498 499 __inline static struct pv_entry * 500 pmap_alloc_pv(pmap, mode) 501 struct pmap *pmap; 502 int mode; 503 { 504 struct pv_page *pvpage; 505 struct pv_entry *pv; 506 507 simple_lock(&pvalloc_lock); 508 509 if (pv_freepages.tqh_first != NULL) { 510 pvpage = pv_freepages.tqh_first; 511 pvpage->pvinfo.pvpi_nfree--; 512 if (pvpage->pvinfo.pvpi_nfree == 0) { 513 /* nothing left in this one? */ 514 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 515 } 516 pv = pvpage->pvinfo.pvpi_pvfree; 517 #ifdef DIAGNOSTIC 518 if (pv == NULL) 519 panic("pmap_alloc_pv: pvpi_nfree off"); 520 #endif 521 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 522 pv_nfpvents--; /* took one from pool */ 523 } else { 524 pv = NULL; /* need more of them */ 525 } 526 527 /* 528 * if below low water mark or we didn't get a pv_entry we try and 529 * create more pv_entrys ... 530 */ 531 532 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 533 if (pv == NULL) 534 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 535 mode : ALLOCPV_NEED); 536 else 537 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 538 } 539 540 simple_unlock(&pvalloc_lock); 541 return(pv); 542 } 543 544 /* 545 * pmap_alloc_pvpage: maybe allocate a new pvpage 546 * 547 * if need_entry is false: try and allocate a new pv_page 548 * if need_entry is true: try and allocate a new pv_page and return a 549 * new pv_entry from it. if we are unable to allocate a pv_page 550 * we make a last ditch effort to steal a pv_page from some other 551 * mapping. if that fails, we panic... 552 * 553 * => we assume that the caller holds pvalloc_lock 554 */ 555 556 static struct pv_entry * 557 pmap_alloc_pvpage(pmap, mode) 558 struct pmap *pmap; 559 int mode; 560 { 561 struct vm_page *pg; 562 struct pv_page *pvpage; 563 struct pv_entry *pv; 564 int s; 565 566 /* 567 * if we need_entry and we've got unused pv_pages, allocate from there 568 */ 569 570 if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) { 571 572 /* move it to pv_freepages list */ 573 pvpage = pv_unusedpgs.tqh_first; 574 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 575 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 576 577 /* allocate a pv_entry */ 578 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 579 pv = pvpage->pvinfo.pvpi_pvfree; 580 #ifdef DIAGNOSTIC 581 if (pv == NULL) 582 panic("pmap_alloc_pvpage: pvpi_nfree off"); 583 #endif 584 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 585 586 pv_nfpvents--; /* took one from pool */ 587 return(pv); 588 } 589 590 /* 591 * see if we've got a cached unmapped VA that we can map a page in. 592 * if not, try to allocate one. 593 */ 594 595 596 if (pv_cachedva == 0) { 597 s = splvm(); 598 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 599 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 600 splx(s); 601 if (pv_cachedva == 0) { 602 return (NULL); 603 } 604 } 605 606 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 607 UVM_PGA_USERESERVE); 608 if (pg) 609 pg->flags &= ~PG_BUSY; /* never busy */ 610 611 if (pg == NULL) 612 return (NULL); 613 614 /* 615 * add a mapping for our new pv_page and free its entrys (save one!) 616 * 617 * NOTE: If we are allocating a PV page for the kernel pmap, the 618 * pmap is already locked! (...but entering the mapping is safe...) 619 */ 620 621 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL); 622 pmap_update(pmap_kernel()); 623 pvpage = (struct pv_page *) pv_cachedva; 624 pv_cachedva = 0; 625 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 626 } 627 628 /* 629 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 630 * 631 * => caller must hold pvalloc_lock 632 * => if need_entry is true, we allocate and return one pv_entry 633 */ 634 635 static struct pv_entry * 636 pmap_add_pvpage(pvp, need_entry) 637 struct pv_page *pvp; 638 boolean_t need_entry; 639 { 640 int tofree, lcv; 641 642 /* do we need to return one? */ 643 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 644 645 pvp->pvinfo.pvpi_pvfree = NULL; 646 pvp->pvinfo.pvpi_nfree = tofree; 647 for (lcv = 0 ; lcv < tofree ; lcv++) { 648 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 649 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 650 } 651 if (need_entry) 652 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 653 else 654 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 655 pv_nfpvents += tofree; 656 return((need_entry) ? &pvp->pvents[lcv] : NULL); 657 } 658 659 /* 660 * pmap_free_pv_doit: actually free a pv_entry 661 * 662 * => do not call this directly! instead use either 663 * 1. pmap_free_pv ==> free a single pv_entry 664 * 2. pmap_free_pvs => free a list of pv_entrys 665 * => we must be holding pvalloc_lock 666 */ 667 668 __inline static void 669 pmap_free_pv_doit(pv) 670 struct pv_entry *pv; 671 { 672 struct pv_page *pvp; 673 674 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 675 pv_nfpvents++; 676 pvp->pvinfo.pvpi_nfree++; 677 678 /* nfree == 1 => fully allocated page just became partly allocated */ 679 if (pvp->pvinfo.pvpi_nfree == 1) { 680 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 681 } 682 683 /* free it */ 684 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 685 pvp->pvinfo.pvpi_pvfree = pv; 686 687 /* 688 * are all pv_page's pv_entry's free? move it to unused queue. 689 */ 690 691 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 692 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 693 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 694 } 695 } 696 697 /* 698 * pmap_free_pv: free a single pv_entry 699 * 700 * => we gain the pvalloc_lock 701 */ 702 703 __inline static void 704 pmap_free_pv(pmap, pv) 705 struct pmap *pmap; 706 struct pv_entry *pv; 707 { 708 simple_lock(&pvalloc_lock); 709 pmap_free_pv_doit(pv); 710 711 /* 712 * Can't free the PV page if the PV entries were associated with 713 * the kernel pmap; the pmap is already locked. 714 */ 715 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 716 pmap != pmap_kernel()) 717 pmap_free_pvpage(); 718 719 simple_unlock(&pvalloc_lock); 720 } 721 722 /* 723 * pmap_free_pvs: free a list of pv_entrys 724 * 725 * => we gain the pvalloc_lock 726 */ 727 728 __inline static void 729 pmap_free_pvs(pmap, pvs) 730 struct pmap *pmap; 731 struct pv_entry *pvs; 732 { 733 struct pv_entry *nextpv; 734 735 simple_lock(&pvalloc_lock); 736 737 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 738 nextpv = pvs->pv_next; 739 pmap_free_pv_doit(pvs); 740 } 741 742 /* 743 * Can't free the PV page if the PV entries were associated with 744 * the kernel pmap; the pmap is already locked. 745 */ 746 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 747 pmap != pmap_kernel()) 748 pmap_free_pvpage(); 749 750 simple_unlock(&pvalloc_lock); 751 } 752 753 754 /* 755 * pmap_free_pvpage: try and free an unused pv_page structure 756 * 757 * => assume caller is holding the pvalloc_lock and that 758 * there is a page on the pv_unusedpgs list 759 * => if we can't get a lock on the kmem_map we try again later 760 * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows 761 * that if we can lock the kmem_map then we are not already 762 * holding kmem_object's lock. 763 */ 764 765 static void 766 pmap_free_pvpage() 767 { 768 int s; 769 struct vm_map *map; 770 struct vm_map_entry *dead_entries; 771 struct pv_page *pvp; 772 773 s = splvm(); /* protect kmem_map */ 774 775 pvp = pv_unusedpgs.tqh_first; 776 777 /* 778 * note: watch out for pv_initpage which is allocated out of 779 * kernel_map rather than kmem_map. 780 */ 781 if (pvp == pv_initpage) 782 map = kernel_map; 783 else 784 map = kmem_map; 785 786 if (vm_map_lock_try(map)) { 787 788 /* remove pvp from pv_unusedpgs */ 789 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 790 791 /* unmap the page */ 792 dead_entries = NULL; 793 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 794 &dead_entries); 795 vm_map_unlock(map); 796 797 if (dead_entries != NULL) 798 uvm_unmap_detach(dead_entries, 0); 799 800 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 801 } 802 803 if (pvp == pv_initpage) 804 /* no more initpage, we've freed it */ 805 pv_initpage = NULL; 806 807 splx(s); 808 } 809 810 /* 811 * main pv_entry manipulation functions: 812 * pmap_enter_pv: enter a mapping onto a vm_page list 813 * pmap_remove_pv: remove a mappiing from a vm_page list 814 * 815 * NOTE: pmap_enter_pv expects to lock the pvh itself 816 * pmap_remove_pv expects te caller to lock the pvh before calling 817 */ 818 819 /* 820 * pmap_enter_pv: enter a mapping onto a vm_page lst 821 * 822 * => caller should hold the proper lock on pmap_main_lock 823 * => caller should have pmap locked 824 * => we will gain the lock on the vm_page and allocate the new pv_entry 825 * => caller should adjust ptp's wire_count before calling 826 * => caller should not adjust pmap's wire_count 827 */ 828 829 __inline static void 830 pmap_enter_pv(pg, pve, pmap, va, ptp, flags) 831 struct vm_page *pg; 832 struct pv_entry *pve; /* preallocated pve for us to use */ 833 struct pmap *pmap; 834 vaddr_t va; 835 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 836 int flags; 837 { 838 pve->pv_pmap = pmap; 839 pve->pv_va = va; 840 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 841 pve->pv_flags = flags; 842 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 843 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 844 pg->mdpage.pvh_list = pve; /* ... locked list */ 845 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 846 if (pve->pv_flags & PT_W) 847 ++pmap->pm_stats.wired_count; 848 } 849 850 /* 851 * pmap_remove_pv: try to remove a mapping from a pv_list 852 * 853 * => caller should hold proper lock on pmap_main_lock 854 * => pmap should be locked 855 * => caller should hold lock on vm_page [so that attrs can be adjusted] 856 * => caller should adjust ptp's wire_count and free PTP if needed 857 * => caller should NOT adjust pmap's wire_count 858 * => we return the removed pve 859 */ 860 861 __inline static struct pv_entry * 862 pmap_remove_pv(pg, pmap, va) 863 struct vm_page *pg; 864 struct pmap *pmap; 865 vaddr_t va; 866 { 867 struct pv_entry *pve, **prevptr; 868 869 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 870 pve = *prevptr; 871 while (pve) { 872 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 873 *prevptr = pve->pv_next; /* remove it! */ 874 if (pve->pv_flags & PT_W) 875 --pmap->pm_stats.wired_count; 876 break; 877 } 878 prevptr = &pve->pv_next; /* previous pointer */ 879 pve = pve->pv_next; /* advance */ 880 } 881 return(pve); /* return removed pve */ 882 } 883 884 /* 885 * 886 * pmap_modify_pv: Update pv flags 887 * 888 * => caller should hold lock on vm_page [so that attrs can be adjusted] 889 * => caller should NOT adjust pmap's wire_count 890 * => caller must call pmap_vac_me_harder() if writable status of a page 891 * may have changed. 892 * => we return the old flags 893 * 894 * Modify a physical-virtual mapping in the pv table 895 */ 896 897 /*__inline */ 898 static u_int 899 pmap_modify_pv(pmap, va, pg, bic_mask, eor_mask) 900 struct pmap *pmap; 901 vaddr_t va; 902 struct vm_page *pg; 903 u_int bic_mask; 904 u_int eor_mask; 905 { 906 struct pv_entry *npv; 907 u_int flags, oflags; 908 909 /* 910 * There is at least one VA mapping this page. 911 */ 912 913 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 914 if (pmap == npv->pv_pmap && va == npv->pv_va) { 915 oflags = npv->pv_flags; 916 npv->pv_flags = flags = 917 ((oflags & ~bic_mask) ^ eor_mask); 918 if ((flags ^ oflags) & PT_W) { 919 if (flags & PT_W) 920 ++pmap->pm_stats.wired_count; 921 else 922 --pmap->pm_stats.wired_count; 923 } 924 return (oflags); 925 } 926 } 927 return (0); 928 } 929 930 /* 931 * Map the specified level 2 pagetable into the level 1 page table for 932 * the given pmap to cover a chunk of virtual address space starting from the 933 * address specified. 934 */ 935 static /*__inline*/ void 936 pmap_map_in_l1(pmap, va, l2pa, selfref) 937 struct pmap *pmap; 938 vaddr_t va, l2pa; 939 boolean_t selfref; 940 { 941 vaddr_t ptva; 942 943 /* Calculate the index into the L1 page table. */ 944 ptva = (va >> PDSHIFT) & ~3; 945 946 NPDEBUG(PDB_MAP_L1, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 947 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 948 949 /* Map page table into the L1. */ 950 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 951 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 952 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 953 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 954 955 /* Map the page table into the page table area. */ 956 if (selfref) { 957 NPDEBUG(PDB_MAP_L1, printf("pt self reference %lx in %lx\n", 958 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 959 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 960 L2_PTE_NC_NB(l2pa, AP_KRW); 961 } 962 /* XXX should be a purge */ 963 /* cpu_tlb_flushD();*/ 964 } 965 966 #if 0 967 static /*__inline*/ void 968 pmap_unmap_in_l1(pmap, va) 969 struct pmap *pmap; 970 vaddr_t va; 971 { 972 vaddr_t ptva; 973 974 /* Calculate the index into the L1 page table. */ 975 ptva = (va >> PDSHIFT) & ~3; 976 977 /* Unmap page table from the L1. */ 978 pmap->pm_pdir[ptva + 0] = 0; 979 pmap->pm_pdir[ptva + 1] = 0; 980 pmap->pm_pdir[ptva + 2] = 0; 981 pmap->pm_pdir[ptva + 3] = 0; 982 983 /* Unmap the page table from the page table area. */ 984 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 985 986 /* XXX should be a purge */ 987 /* cpu_tlb_flushD();*/ 988 } 989 #endif 990 991 /* 992 * Used to map a range of physical addresses into kernel 993 * virtual address space. 994 * 995 * For now, VM is already on, we only need to map the 996 * specified memory. 997 */ 998 vaddr_t 999 pmap_map(va, spa, epa, prot) 1000 vaddr_t va, spa, epa; 1001 int prot; 1002 { 1003 while (spa < epa) { 1004 pmap_kenter_pa(va, spa, prot); 1005 va += NBPG; 1006 spa += NBPG; 1007 } 1008 pmap_update(pmap_kernel()); 1009 return(va); 1010 } 1011 1012 1013 /* 1014 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1015 * 1016 * bootstrap the pmap system. This is called from initarm and allows 1017 * the pmap system to initailise any structures it requires. 1018 * 1019 * Currently this sets up the kernel_pmap that is statically allocated 1020 * and also allocated virtual addresses for certain page hooks. 1021 * Currently the only one page hook is allocated that is used 1022 * to zero physical pages of memory. 1023 * It also initialises the start and end address of the kernel data space. 1024 */ 1025 extern paddr_t physical_freestart; 1026 extern paddr_t physical_freeend; 1027 1028 char *boot_head; 1029 1030 void 1031 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1032 pd_entry_t *kernel_l1pt; 1033 pv_addr_t kernel_ptpt; 1034 { 1035 int loop; 1036 paddr_t start, end; 1037 #if NISADMA > 0 1038 paddr_t istart; 1039 psize_t isize; 1040 #endif 1041 1042 pmap_kernel()->pm_pdir = kernel_l1pt; 1043 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1044 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1045 simple_lock_init(&pmap_kernel()->pm_lock); 1046 pmap_kernel()->pm_obj.pgops = NULL; 1047 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1048 pmap_kernel()->pm_obj.uo_npages = 0; 1049 pmap_kernel()->pm_obj.uo_refs = 1; 1050 1051 /* 1052 * Initialize PAGE_SIZE-dependent variables. 1053 */ 1054 uvm_setpagesize(); 1055 1056 npages = 0; 1057 loop = 0; 1058 while (loop < bootconfig.dramblocks) { 1059 start = (paddr_t)bootconfig.dram[loop].address; 1060 end = start + (bootconfig.dram[loop].pages * NBPG); 1061 if (start < physical_freestart) 1062 start = physical_freestart; 1063 if (end > physical_freeend) 1064 end = physical_freeend; 1065 #if 0 1066 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1067 #endif 1068 #if NISADMA > 0 1069 if (pmap_isa_dma_range_intersect(start, end - start, 1070 &istart, &isize)) { 1071 /* 1072 * Place the pages that intersect with the 1073 * ISA DMA range onto the ISA DMA free list. 1074 */ 1075 #if 0 1076 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1077 istart + isize - 1); 1078 #endif 1079 uvm_page_physload(atop(istart), 1080 atop(istart + isize), atop(istart), 1081 atop(istart + isize), VM_FREELIST_ISADMA); 1082 npages += atop(istart + isize) - atop(istart); 1083 1084 /* 1085 * Load the pieces that come before 1086 * the intersection into the default 1087 * free list. 1088 */ 1089 if (start < istart) { 1090 #if 0 1091 printf(" BEFORE 0x%lx -> 0x%lx\n", 1092 start, istart - 1); 1093 #endif 1094 uvm_page_physload(atop(start), 1095 atop(istart), atop(start), 1096 atop(istart), VM_FREELIST_DEFAULT); 1097 npages += atop(istart) - atop(start); 1098 } 1099 1100 /* 1101 * Load the pieces that come after 1102 * the intersection into the default 1103 * free list. 1104 */ 1105 if ((istart + isize) < end) { 1106 #if 0 1107 printf(" AFTER 0x%lx -> 0x%lx\n", 1108 (istart + isize), end - 1); 1109 #endif 1110 uvm_page_physload(atop(istart + isize), 1111 atop(end), atop(istart + isize), 1112 atop(end), VM_FREELIST_DEFAULT); 1113 npages += atop(end) - atop(istart + isize); 1114 } 1115 } else { 1116 uvm_page_physload(atop(start), atop(end), 1117 atop(start), atop(end), VM_FREELIST_DEFAULT); 1118 npages += atop(end) - atop(start); 1119 } 1120 #else /* NISADMA > 0 */ 1121 uvm_page_physload(atop(start), atop(end), 1122 atop(start), atop(end), VM_FREELIST_DEFAULT); 1123 npages += atop(end) - atop(start); 1124 #endif /* NISADMA > 0 */ 1125 ++loop; 1126 } 1127 1128 #ifdef MYCROFT_HACK 1129 printf("npages = %ld\n", npages); 1130 #endif 1131 1132 virtual_start = KERNEL_VM_BASE; 1133 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE - 1; 1134 1135 ALLOC_PAGE_HOOK(page_hook0, NBPG); 1136 ALLOC_PAGE_HOOK(page_hook1, NBPG); 1137 1138 /* 1139 * The mem special device needs a virtual hook but we don't 1140 * need a pte 1141 */ 1142 memhook = (char *)virtual_start; 1143 virtual_start += NBPG; 1144 1145 msgbufaddr = (caddr_t)virtual_start; 1146 msgbufpte = (pt_entry_t)pmap_pte(pmap_kernel(), virtual_start); 1147 virtual_start += round_page(MSGBUFSIZE); 1148 1149 /* 1150 * init the static-global locks and global lists. 1151 */ 1152 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1153 simple_lock_init(&pvalloc_lock); 1154 simple_lock_init(&pmaps_lock); 1155 LIST_INIT(&pmaps); 1156 TAILQ_INIT(&pv_freepages); 1157 TAILQ_INIT(&pv_unusedpgs); 1158 1159 /* 1160 * initialize the pmap pool. 1161 */ 1162 1163 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1164 0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP); 1165 1166 cpu_dcache_wbinv_all(); 1167 } 1168 1169 /* 1170 * void pmap_init(void) 1171 * 1172 * Initialize the pmap module. 1173 * Called by vm_init() in vm/vm_init.c in order to initialise 1174 * any structures that the pmap system needs to map virtual memory. 1175 */ 1176 1177 extern int physmem; 1178 1179 void 1180 pmap_init() 1181 { 1182 1183 /* 1184 * Set the available memory vars - These do not map to real memory 1185 * addresses and cannot as the physical memory is fragmented. 1186 * They are used by ps for %mem calculations. 1187 * One could argue whether this should be the entire memory or just 1188 * the memory that is useable in a user process. 1189 */ 1190 avail_start = 0; 1191 avail_end = physmem * NBPG; 1192 1193 /* 1194 * now we need to free enough pv_entry structures to allow us to get 1195 * the kmem_map/kmem_object allocated and inited (done after this 1196 * function is finished). to do this we allocate one bootstrap page out 1197 * of kernel_map and use it to provide an initial pool of pv_entry 1198 * structures. we never free this page. 1199 */ 1200 1201 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1202 if (pv_initpage == NULL) 1203 panic("pmap_init: pv_initpage"); 1204 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1205 pv_nfpvents = 0; 1206 (void) pmap_add_pvpage(pv_initpage, FALSE); 1207 1208 pmap_initialized = TRUE; 1209 1210 /* Initialise our L1 page table queues and counters */ 1211 SIMPLEQ_INIT(&l1pt_static_queue); 1212 l1pt_static_queue_count = 0; 1213 l1pt_static_create_count = 0; 1214 SIMPLEQ_INIT(&l1pt_queue); 1215 l1pt_queue_count = 0; 1216 l1pt_create_count = 0; 1217 l1pt_reuse_count = 0; 1218 } 1219 1220 /* 1221 * pmap_postinit() 1222 * 1223 * This routine is called after the vm and kmem subsystems have been 1224 * initialised. This allows the pmap code to perform any initialisation 1225 * that can only be done one the memory allocation is in place. 1226 */ 1227 1228 void 1229 pmap_postinit() 1230 { 1231 int loop; 1232 struct l1pt *pt; 1233 1234 #ifdef PMAP_STATIC_L1S 1235 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1236 #else /* PMAP_STATIC_L1S */ 1237 for (loop = 0; loop < max_processes; ++loop) { 1238 #endif /* PMAP_STATIC_L1S */ 1239 /* Allocate a L1 page table */ 1240 pt = pmap_alloc_l1pt(); 1241 if (!pt) 1242 panic("Cannot allocate static L1 page tables\n"); 1243 1244 /* Clean it */ 1245 bzero((void *)pt->pt_va, PD_SIZE); 1246 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1247 /* Add the page table to the queue */ 1248 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1249 ++l1pt_static_queue_count; 1250 ++l1pt_static_create_count; 1251 } 1252 } 1253 1254 1255 /* 1256 * Create and return a physical map. 1257 * 1258 * If the size specified for the map is zero, the map is an actual physical 1259 * map, and may be referenced by the hardware. 1260 * 1261 * If the size specified is non-zero, the map will be used in software only, 1262 * and is bounded by that size. 1263 */ 1264 1265 pmap_t 1266 pmap_create() 1267 { 1268 struct pmap *pmap; 1269 1270 /* 1271 * Fetch pmap entry from the pool 1272 */ 1273 1274 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1275 /* XXX is this really needed! */ 1276 memset(pmap, 0, sizeof(*pmap)); 1277 1278 simple_lock_init(&pmap->pm_obj.vmobjlock); 1279 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1280 TAILQ_INIT(&pmap->pm_obj.memq); 1281 pmap->pm_obj.uo_npages = 0; 1282 pmap->pm_obj.uo_refs = 1; 1283 pmap->pm_stats.wired_count = 0; 1284 pmap->pm_stats.resident_count = 1; 1285 1286 /* Now init the machine part of the pmap */ 1287 pmap_pinit(pmap); 1288 return(pmap); 1289 } 1290 1291 /* 1292 * pmap_alloc_l1pt() 1293 * 1294 * This routine allocates physical and virtual memory for a L1 page table 1295 * and wires it. 1296 * A l1pt structure is returned to describe the allocated page table. 1297 * 1298 * This routine is allowed to fail if the required memory cannot be allocated. 1299 * In this case NULL is returned. 1300 */ 1301 1302 struct l1pt * 1303 pmap_alloc_l1pt(void) 1304 { 1305 paddr_t pa; 1306 vaddr_t va; 1307 struct l1pt *pt; 1308 int error; 1309 struct vm_page *m; 1310 pt_entry_t *ptes; 1311 1312 /* Allocate virtual address space for the L1 page table */ 1313 va = uvm_km_valloc(kernel_map, PD_SIZE); 1314 if (va == 0) { 1315 #ifdef DIAGNOSTIC 1316 PDEBUG(0, 1317 printf("pmap: Cannot allocate pageable memory for L1\n")); 1318 #endif /* DIAGNOSTIC */ 1319 return(NULL); 1320 } 1321 1322 /* Allocate memory for the l1pt structure */ 1323 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1324 1325 /* 1326 * Allocate pages from the VM system. 1327 */ 1328 TAILQ_INIT(&pt->pt_plist); 1329 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1330 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1331 if (error) { 1332 #ifdef DIAGNOSTIC 1333 PDEBUG(0, 1334 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1335 error)); 1336 #endif /* DIAGNOSTIC */ 1337 /* Release the resources we already have claimed */ 1338 free(pt, M_VMPMAP); 1339 uvm_km_free(kernel_map, va, PD_SIZE); 1340 return(NULL); 1341 } 1342 1343 /* Map our physical pages into our virtual space */ 1344 pt->pt_va = va; 1345 m = pt->pt_plist.tqh_first; 1346 ptes = pmap_map_ptes(pmap_kernel()); 1347 while (m && va < (pt->pt_va + PD_SIZE)) { 1348 pa = VM_PAGE_TO_PHYS(m); 1349 1350 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1351 1352 /* Revoke cacheability and bufferability */ 1353 /* XXX should be done better than this */ 1354 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1355 1356 va += NBPG; 1357 m = m->pageq.tqe_next; 1358 } 1359 pmap_unmap_ptes(pmap_kernel()); 1360 pmap_update(pmap_kernel()); 1361 1362 #ifdef DIAGNOSTIC 1363 if (m) 1364 panic("pmap_alloc_l1pt: pglist not empty\n"); 1365 #endif /* DIAGNOSTIC */ 1366 1367 pt->pt_flags = 0; 1368 return(pt); 1369 } 1370 1371 /* 1372 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1373 */ 1374 static void 1375 pmap_free_l1pt(pt) 1376 struct l1pt *pt; 1377 { 1378 /* Separate the physical memory for the virtual space */ 1379 pmap_kremove(pt->pt_va, PD_SIZE); 1380 pmap_update(pmap_kernel()); 1381 1382 /* Return the physical memory */ 1383 uvm_pglistfree(&pt->pt_plist); 1384 1385 /* Free the virtual space */ 1386 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1387 1388 /* Free the l1pt structure */ 1389 free(pt, M_VMPMAP); 1390 } 1391 1392 /* 1393 * Allocate a page directory. 1394 * This routine will either allocate a new page directory from the pool 1395 * of L1 page tables currently held by the kernel or it will allocate 1396 * a new one via pmap_alloc_l1pt(). 1397 * It will then initialise the l1 page table for use. 1398 * 1399 * XXX must tidy up and fix this code, not happy about how it does the pmaps_locking 1400 */ 1401 static int 1402 pmap_allocpagedir(pmap) 1403 struct pmap *pmap; 1404 { 1405 paddr_t pa; 1406 struct l1pt *pt; 1407 pt_entry_t *pte; 1408 1409 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1410 1411 /* Do we have any spare L1's lying around ? */ 1412 if (l1pt_static_queue_count) { 1413 --l1pt_static_queue_count; 1414 pt = l1pt_static_queue.sqh_first; 1415 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1416 } else if (l1pt_queue_count) { 1417 --l1pt_queue_count; 1418 pt = l1pt_queue.sqh_first; 1419 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1420 ++l1pt_reuse_count; 1421 } else { 1422 pt = pmap_alloc_l1pt(); 1423 if (!pt) 1424 return(ENOMEM); 1425 ++l1pt_create_count; 1426 } 1427 1428 /* Store the pointer to the l1 descriptor in the pmap. */ 1429 pmap->pm_l1pt = pt; 1430 1431 /* Get the physical address of the start of the l1 */ 1432 pa = VM_PAGE_TO_PHYS(pt->pt_plist.tqh_first); 1433 1434 /* Store the virtual address of the l1 in the pmap. */ 1435 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1436 1437 /* Clean the L1 if it is dirty */ 1438 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1439 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1440 1441 /* Allocate a page table to map all the page tables for this pmap */ 1442 1443 #ifdef DIAGNOSTIC 1444 if (pmap->pm_vptpt) { 1445 /* XXX What if we have one already ? */ 1446 panic("pmap_allocpagedir: have pt already\n"); 1447 } 1448 #endif /* DIAGNOSTIC */ 1449 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1450 if (pmap->pm_vptpt == 0) { 1451 pmap_freepagedir(pmap); 1452 return(ENOMEM); 1453 } 1454 1455 /* need to lock this all up for growkernel */ 1456 simple_lock(&pmaps_lock); 1457 /* wish we didn't have to keep this locked... */ 1458 1459 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1460 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1461 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1462 KERNEL_PD_SIZE); 1463 1464 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1465 pmap->pm_pptpt &= PG_FRAME; 1466 /* Revoke cacheability and bufferability */ 1467 /* XXX should be done better than this */ 1468 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1469 *pte = *pte & ~(PT_C | PT_B); 1470 1471 /* Wire in this page table */ 1472 pmap_map_in_l1(pmap, PROCESS_PAGE_TBLS_BASE, pmap->pm_pptpt, TRUE); 1473 1474 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1475 1476 /* 1477 * Map the kernel page tables for 0xf0000000 + 1478 * into the page table used to map the 1479 * pmap's page tables 1480 */ 1481 bcopy((char *)(PROCESS_PAGE_TBLS_BASE 1482 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) 1483 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1484 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1485 (KERNEL_PD_SIZE >> 2)); 1486 1487 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1488 simple_unlock(&pmaps_lock); 1489 1490 return(0); 1491 } 1492 1493 1494 /* 1495 * Initialize a preallocated and zeroed pmap structure, 1496 * such as one in a vmspace structure. 1497 */ 1498 1499 void 1500 pmap_pinit(pmap) 1501 struct pmap *pmap; 1502 { 1503 int backoff = 6; 1504 int retry = 10; 1505 1506 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1507 1508 /* Keep looping until we succeed in allocating a page directory */ 1509 while (pmap_allocpagedir(pmap) != 0) { 1510 /* 1511 * Ok we failed to allocate a suitable block of memory for an 1512 * L1 page table. This means that either: 1513 * 1. 16KB of virtual address space could not be allocated 1514 * 2. 16KB of physically contiguous memory on a 16KB boundary 1515 * could not be allocated. 1516 * 1517 * Since we cannot fail we will sleep for a while and try 1518 * again. 1519 * 1520 * Searching for a suitable L1 PT is expensive: 1521 * to avoid hogging the system when memory is really 1522 * scarce, use an exponential back-off so that 1523 * eventually we won't retry more than once every 8 1524 * seconds. This should allow other processes to run 1525 * to completion and free up resources. 1526 */ 1527 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1528 NULL); 1529 if (--retry == 0) { 1530 retry = 10; 1531 if (backoff) 1532 --backoff; 1533 } 1534 } 1535 1536 /* Map zero page for the pmap. This will also map the L2 for it */ 1537 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1538 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1539 pmap_update(pmap); 1540 } 1541 1542 1543 void 1544 pmap_freepagedir(pmap) 1545 struct pmap *pmap; 1546 { 1547 /* Free the memory used for the page table mapping */ 1548 if (pmap->pm_vptpt != 0) 1549 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1550 1551 /* junk the L1 page table */ 1552 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1553 /* Add the page table to the queue */ 1554 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1555 ++l1pt_static_queue_count; 1556 } else if (l1pt_queue_count < 8) { 1557 /* Add the page table to the queue */ 1558 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1559 ++l1pt_queue_count; 1560 } else 1561 pmap_free_l1pt(pmap->pm_l1pt); 1562 } 1563 1564 1565 /* 1566 * Retire the given physical map from service. 1567 * Should only be called if the map contains no valid mappings. 1568 */ 1569 1570 void 1571 pmap_destroy(pmap) 1572 struct pmap *pmap; 1573 { 1574 struct vm_page *page; 1575 int count; 1576 1577 if (pmap == NULL) 1578 return; 1579 1580 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1581 1582 /* 1583 * Drop reference count 1584 */ 1585 simple_lock(&pmap->pm_obj.vmobjlock); 1586 count = --pmap->pm_obj.uo_refs; 1587 simple_unlock(&pmap->pm_obj.vmobjlock); 1588 if (count > 0) { 1589 return; 1590 } 1591 1592 /* 1593 * reference count is zero, free pmap resources and then free pmap. 1594 */ 1595 1596 /* 1597 * remove it from global list of pmaps 1598 */ 1599 1600 simple_lock(&pmaps_lock); 1601 LIST_REMOVE(pmap, pm_list); 1602 simple_unlock(&pmaps_lock); 1603 1604 /* Remove the zero page mapping */ 1605 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1606 pmap_update(pmap); 1607 1608 /* 1609 * Free any page tables still mapped 1610 * This is only temporay until pmap_enter can count the number 1611 * of mappings made in a page table. Then pmap_remove() can 1612 * reduce the count and free the pagetable when the count 1613 * reaches zero. Note that entries in this list should match the 1614 * contents of the ptpt, however this is faster than walking a 1024 1615 * entries looking for pt's 1616 * taken from i386 pmap.c 1617 */ 1618 while (pmap->pm_obj.memq.tqh_first != NULL) { 1619 page = pmap->pm_obj.memq.tqh_first; 1620 #ifdef DIAGNOSTIC 1621 if (page->flags & PG_BUSY) 1622 panic("pmap_release: busy page table page"); 1623 #endif 1624 /* pmap_page_protect? currently no need for it. */ 1625 1626 page->wire_count = 0; 1627 uvm_pagefree(page); 1628 } 1629 1630 /* Free the page dir */ 1631 pmap_freepagedir(pmap); 1632 1633 /* return the pmap to the pool */ 1634 pool_put(&pmap_pmap_pool, pmap); 1635 } 1636 1637 1638 /* 1639 * void pmap_reference(struct pmap *pmap) 1640 * 1641 * Add a reference to the specified pmap. 1642 */ 1643 1644 void 1645 pmap_reference(pmap) 1646 struct pmap *pmap; 1647 { 1648 if (pmap == NULL) 1649 return; 1650 1651 simple_lock(&pmap->pm_lock); 1652 pmap->pm_obj.uo_refs++; 1653 simple_unlock(&pmap->pm_lock); 1654 } 1655 1656 /* 1657 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1658 * 1659 * Return the start and end addresses of the kernel's virtual space. 1660 * These values are setup in pmap_bootstrap and are updated as pages 1661 * are allocated. 1662 */ 1663 1664 void 1665 pmap_virtual_space(start, end) 1666 vaddr_t *start; 1667 vaddr_t *end; 1668 { 1669 *start = virtual_start; 1670 *end = virtual_end; 1671 } 1672 1673 1674 /* 1675 * Activate the address space for the specified process. If the process 1676 * is the current process, load the new MMU context. 1677 */ 1678 void 1679 pmap_activate(p) 1680 struct proc *p; 1681 { 1682 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1683 struct pcb *pcb = &p->p_addr->u_pcb; 1684 1685 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1686 (paddr_t *)&pcb->pcb_pagedir); 1687 1688 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1689 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1690 1691 if (p == curproc) { 1692 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1693 setttb((u_int)pcb->pcb_pagedir); 1694 } 1695 #if 0 1696 pmap->pm_pdchanged = FALSE; 1697 #endif 1698 } 1699 1700 1701 /* 1702 * Deactivate the address space of the specified process. 1703 */ 1704 void 1705 pmap_deactivate(p) 1706 struct proc *p; 1707 { 1708 } 1709 1710 /* 1711 * Perform any deferred pmap operations. 1712 */ 1713 void 1714 pmap_update(struct pmap *pmap) 1715 { 1716 1717 /* 1718 * We haven't deferred any pmap operations, but we do need to 1719 * make sure TLB/cache operations have completed. 1720 */ 1721 cpu_cpwait(); 1722 } 1723 1724 /* 1725 * pmap_clean_page() 1726 * 1727 * This is a local function used to work out the best strategy to clean 1728 * a single page referenced by its entry in the PV table. It's used by 1729 * pmap_copy_page, pmap_zero page and maybe some others later on. 1730 * 1731 * Its policy is effectively: 1732 * o If there are no mappings, we don't bother doing anything with the cache. 1733 * o If there is one mapping, we clean just that page. 1734 * o If there are multiple mappings, we clean the entire cache. 1735 * 1736 * So that some functions can be further optimised, it returns 0 if it didn't 1737 * clean the entire cache, or 1 if it did. 1738 * 1739 * XXX One bug in this routine is that if the pv_entry has a single page 1740 * mapped at 0x00000000 a whole cache clean will be performed rather than 1741 * just the 1 page. Since this should not occur in everyday use and if it does 1742 * it will just result in not the most efficient clean for the page. 1743 */ 1744 static int 1745 pmap_clean_page(pv, is_src) 1746 struct pv_entry *pv; 1747 boolean_t is_src; 1748 { 1749 struct pmap *pmap; 1750 struct pv_entry *npv; 1751 int cache_needs_cleaning = 0; 1752 vaddr_t page_to_clean = 0; 1753 1754 if (pv == NULL) 1755 /* nothing mapped in so nothing to flush */ 1756 return (0); 1757 1758 /* Since we flush the cache each time we change curproc, we 1759 * only need to flush the page if it is in the current pmap. 1760 */ 1761 if (curproc) 1762 pmap = curproc->p_vmspace->vm_map.pmap; 1763 else 1764 pmap = pmap_kernel(); 1765 1766 for (npv = pv; npv; npv = npv->pv_next) { 1767 if (npv->pv_pmap == pmap) { 1768 /* The page is mapped non-cacheable in 1769 * this map. No need to flush the cache. 1770 */ 1771 if (npv->pv_flags & PT_NC) { 1772 #ifdef DIAGNOSTIC 1773 if (cache_needs_cleaning) 1774 panic("pmap_clean_page: " 1775 "cache inconsistency"); 1776 #endif 1777 break; 1778 } 1779 #if 0 1780 /* This doesn't work, because pmap_protect 1781 doesn't flush changes on pages that it 1782 has write-protected. */ 1783 1784 /* If the page is not writable and this 1785 is the source, then there is no need 1786 to flush it from the cache. */ 1787 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1788 continue; 1789 #endif 1790 if (cache_needs_cleaning){ 1791 page_to_clean = 0; 1792 break; 1793 } 1794 else 1795 page_to_clean = npv->pv_va; 1796 cache_needs_cleaning = 1; 1797 } 1798 } 1799 1800 if (page_to_clean) 1801 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1802 else if (cache_needs_cleaning) { 1803 cpu_idcache_wbinv_all(); 1804 return (1); 1805 } 1806 return (0); 1807 } 1808 1809 /* 1810 * pmap_zero_page() 1811 * 1812 * Zero a given physical page by mapping it at a page hook point. 1813 * In doing the zero page op, the page we zero is mapped cachable, as with 1814 * StrongARM accesses to non-cached pages are non-burst making writing 1815 * _any_ bulk data very slow. 1816 */ 1817 void 1818 pmap_zero_page(phys) 1819 paddr_t phys; 1820 { 1821 struct vm_page *pg; 1822 1823 /* Get an entry for this page, and clean it it. */ 1824 pg = PHYS_TO_VM_PAGE(phys); 1825 simple_lock(&pg->mdpage.pvh_slock); 1826 pmap_clean_page(pg->mdpage.pvh_list, FALSE); 1827 simple_unlock(&pg->mdpage.pvh_slock); 1828 1829 /* 1830 * Hook in the page, zero it, and purge the cache for that 1831 * zeroed page. Invalidate the TLB as needed. 1832 */ 1833 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1834 cpu_tlb_flushD_SE(page_hook0.va); 1835 cpu_cpwait(); 1836 bzero_page(page_hook0.va); 1837 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1838 } 1839 1840 /* pmap_pageidlezero() 1841 * 1842 * The same as above, except that we assume that the page is not 1843 * mapped. This means we never have to flush the cache first. Called 1844 * from the idle loop. 1845 */ 1846 boolean_t 1847 pmap_pageidlezero(phys) 1848 paddr_t phys; 1849 { 1850 int i, *ptr; 1851 boolean_t rv = TRUE; 1852 1853 #ifdef DIAGNOSTIC 1854 struct vm_page *pg; 1855 1856 pg = PHYS_TO_VM_PAGE(phys); 1857 if (pg->mdpage.pvh_list != NULL) 1858 panic("pmap_pageidlezero: zeroing mapped page\n"); 1859 #endif 1860 1861 /* 1862 * Hook in the page, zero it, and purge the cache for that 1863 * zeroed page. Invalidate the TLB as needed. 1864 */ 1865 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1866 cpu_tlb_flushD_SE(page_hook0.va); 1867 cpu_cpwait(); 1868 1869 for (i = 0, ptr = (int *)page_hook0.va; 1870 i < (NBPG / sizeof(int)); i++) { 1871 if (sched_whichqs != 0) { 1872 /* 1873 * A process has become ready. Abort now, 1874 * so we don't keep it waiting while we 1875 * do slow memory access to finish this 1876 * page. 1877 */ 1878 rv = FALSE; 1879 break; 1880 } 1881 *ptr++ = 0; 1882 } 1883 1884 if (rv) 1885 /* 1886 * if we aborted we'll rezero this page again later so don't 1887 * purge it unless we finished it 1888 */ 1889 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1890 return (rv); 1891 } 1892 1893 /* 1894 * pmap_copy_page() 1895 * 1896 * Copy one physical page into another, by mapping the pages into 1897 * hook points. The same comment regarding cachability as in 1898 * pmap_zero_page also applies here. 1899 */ 1900 void 1901 pmap_copy_page(src, dest) 1902 paddr_t src; 1903 paddr_t dest; 1904 { 1905 struct vm_page *src_pg, *dest_pg; 1906 boolean_t cleanedcache; 1907 1908 /* Get PV entries for the pages, and clean them if needed. */ 1909 src_pg = PHYS_TO_VM_PAGE(src); 1910 1911 simple_lock(&src_pg->mdpage.pvh_slock); 1912 cleanedcache = pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1913 simple_unlock(&src_pg->mdpage.pvh_slock); 1914 1915 if (cleanedcache == 0) { 1916 dest_pg = PHYS_TO_VM_PAGE(dest); 1917 simple_lock(&dest_pg->mdpage.pvh_slock); 1918 pmap_clean_page(dest_pg->mdpage.pvh_list, FALSE); 1919 simple_unlock(&dest_pg->mdpage.pvh_slock); 1920 } 1921 /* 1922 * Map the pages into the page hook points, copy them, and purge 1923 * the cache for the appropriate page. Invalidate the TLB 1924 * as required. 1925 */ 1926 *page_hook0.pte = L2_PTE(src & PG_FRAME, AP_KRW); 1927 *page_hook1.pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1928 cpu_tlb_flushD_SE(page_hook0.va); 1929 cpu_tlb_flushD_SE(page_hook1.va); 1930 cpu_cpwait(); 1931 bcopy_page(page_hook0.va, page_hook1.va); 1932 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1933 cpu_dcache_wbinv_range(page_hook1.va, NBPG); 1934 } 1935 1936 #if 0 1937 void 1938 pmap_pte_addref(pmap, va) 1939 struct pmap *pmap; 1940 vaddr_t va; 1941 { 1942 pd_entry_t *pde; 1943 paddr_t pa; 1944 struct vm_page *m; 1945 1946 if (pmap == pmap_kernel()) 1947 return; 1948 1949 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1950 pa = pmap_pte_pa(pde); 1951 m = PHYS_TO_VM_PAGE(pa); 1952 ++m->wire_count; 1953 #ifdef MYCROFT_HACK 1954 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1955 pmap, va, pde, pa, m, m->wire_count); 1956 #endif 1957 } 1958 1959 void 1960 pmap_pte_delref(pmap, va) 1961 struct pmap *pmap; 1962 vaddr_t va; 1963 { 1964 pd_entry_t *pde; 1965 paddr_t pa; 1966 struct vm_page *m; 1967 1968 if (pmap == pmap_kernel()) 1969 return; 1970 1971 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1972 pa = pmap_pte_pa(pde); 1973 m = PHYS_TO_VM_PAGE(pa); 1974 --m->wire_count; 1975 #ifdef MYCROFT_HACK 1976 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1977 pmap, va, pde, pa, m, m->wire_count); 1978 #endif 1979 if (m->wire_count == 0) { 1980 #ifdef MYCROFT_HACK 1981 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1982 pmap, va, pde, pa, m); 1983 #endif 1984 pmap_unmap_in_l1(pmap, va); 1985 uvm_pagefree(m); 1986 --pmap->pm_stats.resident_count; 1987 } 1988 } 1989 #else 1990 #define pmap_pte_addref(pmap, va) 1991 #define pmap_pte_delref(pmap, va) 1992 #endif 1993 1994 /* 1995 * Since we have a virtually indexed cache, we may need to inhibit caching if 1996 * there is more than one mapping and at least one of them is writable. 1997 * Since we purge the cache on every context switch, we only need to check for 1998 * other mappings within the same pmap, or kernel_pmap. 1999 * This function is also called when a page is unmapped, to possibly reenable 2000 * caching on any remaining mappings. 2001 * 2002 * The code implements the following logic, where: 2003 * 2004 * KW = # of kernel read/write pages 2005 * KR = # of kernel read only pages 2006 * UW = # of user read/write pages 2007 * UR = # of user read only pages 2008 * OW = # of user read/write pages in another pmap, then 2009 * 2010 * KC = kernel mapping is cacheable 2011 * UC = user mapping is cacheable 2012 * 2013 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2014 * +--------------------------------------------- 2015 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2016 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2017 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2018 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2019 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2020 * 2021 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2022 */ 2023 __inline static void 2024 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2025 boolean_t clear_cache) 2026 { 2027 if (pmap == pmap_kernel()) 2028 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2029 else 2030 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2031 } 2032 2033 static void 2034 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2035 boolean_t clear_cache) 2036 { 2037 int user_entries = 0; 2038 int user_writable = 0; 2039 int user_cacheable = 0; 2040 int kernel_entries = 0; 2041 int kernel_writable = 0; 2042 int kernel_cacheable = 0; 2043 struct pv_entry *pv; 2044 struct pmap *last_pmap = pmap; 2045 2046 #ifdef DIAGNOSTIC 2047 if (pmap != pmap_kernel()) 2048 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2049 #endif 2050 2051 /* 2052 * Pass one, see if there are both kernel and user pmaps for 2053 * this page. Calculate whether there are user-writable or 2054 * kernel-writable pages. 2055 */ 2056 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2057 if (pv->pv_pmap != pmap) { 2058 user_entries++; 2059 if (pv->pv_flags & PT_Wr) 2060 user_writable++; 2061 if ((pv->pv_flags & PT_NC) == 0) 2062 user_cacheable++; 2063 } else { 2064 kernel_entries++; 2065 if (pv->pv_flags & PT_Wr) 2066 kernel_writable++; 2067 if ((pv->pv_flags & PT_NC) == 0) 2068 kernel_cacheable++; 2069 } 2070 } 2071 2072 /* 2073 * We know we have just been updating a kernel entry, so if 2074 * all user pages are already cacheable, then there is nothing 2075 * further to do. 2076 */ 2077 if (kernel_entries == 0 && 2078 user_cacheable == user_entries) 2079 return; 2080 2081 if (user_entries) { 2082 /* 2083 * Scan over the list again, for each entry, if it 2084 * might not be set correctly, call pmap_vac_me_user 2085 * to recalculate the settings. 2086 */ 2087 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2088 /* 2089 * We know kernel mappings will get set 2090 * correctly in other calls. We also know 2091 * that if the pmap is the same as last_pmap 2092 * then we've just handled this entry. 2093 */ 2094 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2095 continue; 2096 /* 2097 * If there are kernel entries and this page 2098 * is writable but non-cacheable, then we can 2099 * skip this entry also. 2100 */ 2101 if (kernel_entries > 0 && 2102 (pv->pv_flags & (PT_NC | PT_Wr)) == 2103 (PT_NC | PT_Wr)) 2104 continue; 2105 /* 2106 * Similarly if there are no kernel-writable 2107 * entries and the page is already 2108 * read-only/cacheable. 2109 */ 2110 if (kernel_writable == 0 && 2111 (pv->pv_flags & (PT_NC | PT_Wr)) == 0) 2112 continue; 2113 /* 2114 * For some of the remaining cases, we know 2115 * that we must recalculate, but for others we 2116 * can't tell if they are correct or not, so 2117 * we recalculate anyway. 2118 */ 2119 pmap_unmap_ptes(last_pmap); 2120 last_pmap = pv->pv_pmap; 2121 ptes = pmap_map_ptes(last_pmap); 2122 pmap_vac_me_user(last_pmap, pg, ptes, 2123 pmap_is_curpmap(last_pmap)); 2124 } 2125 /* Restore the pte mapping that was passed to us. */ 2126 if (last_pmap != pmap) { 2127 pmap_unmap_ptes(last_pmap); 2128 ptes = pmap_map_ptes(pmap); 2129 } 2130 if (kernel_entries == 0) 2131 return; 2132 } 2133 2134 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2135 return; 2136 } 2137 2138 static void 2139 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2140 boolean_t clear_cache) 2141 { 2142 struct pmap *kpmap = pmap_kernel(); 2143 struct pv_entry *pv, *npv; 2144 int entries = 0; 2145 int writable = 0; 2146 int cacheable_entries = 0; 2147 int kern_cacheable = 0; 2148 int other_writable = 0; 2149 2150 pv = pg->mdpage.pvh_list; 2151 KASSERT(ptes != NULL); 2152 2153 /* 2154 * Count mappings and writable mappings in this pmap. 2155 * Include kernel mappings as part of our own. 2156 * Keep a pointer to the first one. 2157 */ 2158 for (npv = pv; npv; npv = npv->pv_next) { 2159 /* Count mappings in the same pmap */ 2160 if (pmap == npv->pv_pmap || 2161 kpmap == npv->pv_pmap) { 2162 if (entries++ == 0) 2163 pv = npv; 2164 /* Cacheable mappings */ 2165 if ((npv->pv_flags & PT_NC) == 0) { 2166 cacheable_entries++; 2167 if (kpmap == npv->pv_pmap) 2168 kern_cacheable++; 2169 } 2170 /* Writable mappings */ 2171 if (npv->pv_flags & PT_Wr) 2172 ++writable; 2173 } else if (npv->pv_flags & PT_Wr) 2174 other_writable = 1; 2175 } 2176 2177 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2178 "writable %d cacheable %d %s\n", pmap, entries, writable, 2179 cacheable_entries, clear_cache ? "clean" : "no clean")); 2180 2181 /* 2182 * Enable or disable caching as necessary. 2183 * Note: the first entry might be part of the kernel pmap, 2184 * so we can't assume this is indicative of the state of the 2185 * other (maybe non-kpmap) entries. 2186 */ 2187 if ((entries > 1 && writable) || 2188 (entries > 0 && pmap == kpmap && other_writable)) { 2189 if (cacheable_entries == 0) 2190 return; 2191 for (npv = pv; npv; npv = npv->pv_next) { 2192 if ((pmap == npv->pv_pmap 2193 || kpmap == npv->pv_pmap) && 2194 (npv->pv_flags & PT_NC) == 0) { 2195 ptes[arm_byte_to_page(npv->pv_va)] &= 2196 ~(PT_C | PT_B); 2197 npv->pv_flags |= PT_NC; 2198 /* 2199 * If this page needs flushing from the 2200 * cache, and we aren't going to do it 2201 * below, do it now. 2202 */ 2203 if ((cacheable_entries < 4 && 2204 (clear_cache || npv->pv_pmap == kpmap)) || 2205 (npv->pv_pmap == kpmap && 2206 !clear_cache && kern_cacheable < 4)) { 2207 cpu_idcache_wbinv_range(npv->pv_va, 2208 NBPG); 2209 cpu_tlb_flushID_SE(npv->pv_va); 2210 } 2211 } 2212 } 2213 if ((clear_cache && cacheable_entries >= 4) || 2214 kern_cacheable >= 4) { 2215 cpu_idcache_wbinv_all(); 2216 cpu_tlb_flushID(); 2217 } 2218 cpu_cpwait(); 2219 } else if (entries > 0) { 2220 /* 2221 * Turn cacheing back on for some pages. If it is a kernel 2222 * page, only do so if there are no other writable pages. 2223 */ 2224 for (npv = pv; npv; npv = npv->pv_next) { 2225 if ((pmap == npv->pv_pmap || 2226 (kpmap == npv->pv_pmap && other_writable == 0)) && 2227 (npv->pv_flags & PT_NC)) { 2228 ptes[arm_byte_to_page(npv->pv_va)] |= 2229 pte_cache_mode; 2230 npv->pv_flags &= ~PT_NC; 2231 } 2232 } 2233 } 2234 } 2235 2236 /* 2237 * pmap_remove() 2238 * 2239 * pmap_remove is responsible for nuking a number of mappings for a range 2240 * of virtual address space in the current pmap. To do this efficiently 2241 * is interesting, because in a number of cases a wide virtual address 2242 * range may be supplied that contains few actual mappings. So, the 2243 * optimisations are: 2244 * 1. Try and skip over hunks of address space for which an L1 entry 2245 * does not exist. 2246 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2247 * maybe do just a partial cache clean. This path of execution is 2248 * complicated by the fact that the cache must be flushed _before_ 2249 * the PTE is nuked, being a VAC :-) 2250 * 3. Maybe later fast-case a single page, but I don't think this is 2251 * going to make _that_ much difference overall. 2252 */ 2253 2254 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2255 2256 void 2257 pmap_remove(pmap, sva, eva) 2258 struct pmap *pmap; 2259 vaddr_t sva; 2260 vaddr_t eva; 2261 { 2262 int cleanlist_idx = 0; 2263 struct pagelist { 2264 vaddr_t va; 2265 pt_entry_t *pte; 2266 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2267 pt_entry_t *pte = 0, *ptes; 2268 paddr_t pa; 2269 int pmap_active; 2270 struct vm_page *pg; 2271 2272 /* Exit quick if there is no pmap */ 2273 if (!pmap) 2274 return; 2275 2276 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2277 2278 sva &= PG_FRAME; 2279 eva &= PG_FRAME; 2280 2281 /* 2282 * we lock in the pmap => vm_page direction 2283 */ 2284 PMAP_MAP_TO_HEAD_LOCK(); 2285 2286 ptes = pmap_map_ptes(pmap); 2287 /* Get a page table pointer */ 2288 while (sva < eva) { 2289 if (pmap_pde_page(pmap_pde(pmap, sva))) 2290 break; 2291 sva = (sva & PD_MASK) + NBPD; 2292 } 2293 2294 pte = &ptes[arm_byte_to_page(sva)]; 2295 /* Note if the pmap is active thus require cache and tlb cleans */ 2296 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2297 || (pmap == pmap_kernel())) 2298 pmap_active = 1; 2299 else 2300 pmap_active = 0; 2301 2302 /* Now loop along */ 2303 while (sva < eva) { 2304 /* Check if we can move to the next PDE (l1 chunk) */ 2305 if (!(sva & PT_MASK)) 2306 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2307 sva += NBPD; 2308 pte += arm_byte_to_page(NBPD); 2309 continue; 2310 } 2311 2312 /* We've found a valid PTE, so this page of PTEs has to go. */ 2313 if (pmap_pte_v(pte)) { 2314 /* Update statistics */ 2315 --pmap->pm_stats.resident_count; 2316 2317 /* 2318 * Add this page to our cache remove list, if we can. 2319 * If, however the cache remove list is totally full, 2320 * then do a complete cache invalidation taking note 2321 * to backtrack the PTE table beforehand, and ignore 2322 * the lists in future because there's no longer any 2323 * point in bothering with them (we've paid the 2324 * penalty, so will carry on unhindered). Otherwise, 2325 * when we fall out, we just clean the list. 2326 */ 2327 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2328 pa = pmap_pte_pa(pte); 2329 2330 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2331 /* Add to the clean list. */ 2332 cleanlist[cleanlist_idx].pte = pte; 2333 cleanlist[cleanlist_idx].va = sva; 2334 cleanlist_idx++; 2335 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2336 int cnt; 2337 2338 /* Nuke everything if needed. */ 2339 if (pmap_active) { 2340 cpu_idcache_wbinv_all(); 2341 cpu_tlb_flushID(); 2342 } 2343 2344 /* 2345 * Roll back the previous PTE list, 2346 * and zero out the current PTE. 2347 */ 2348 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2349 *cleanlist[cnt].pte = 0; 2350 pmap_pte_delref(pmap, cleanlist[cnt].va); 2351 } 2352 *pte = 0; 2353 pmap_pte_delref(pmap, sva); 2354 cleanlist_idx++; 2355 } else { 2356 /* 2357 * We've already nuked the cache and 2358 * TLB, so just carry on regardless, 2359 * and we won't need to do it again 2360 */ 2361 *pte = 0; 2362 pmap_pte_delref(pmap, sva); 2363 } 2364 2365 /* 2366 * Update flags. In a number of circumstances, 2367 * we could cluster a lot of these and do a 2368 * number of sequential pages in one go. 2369 */ 2370 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2371 struct pv_entry *pve; 2372 simple_lock(&pg->mdpage.pvh_slock); 2373 pve = pmap_remove_pv(pg, pmap, sva); 2374 pmap_free_pv(pmap, pve); 2375 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2376 simple_unlock(&pg->mdpage.pvh_slock); 2377 } 2378 } 2379 sva += NBPG; 2380 pte++; 2381 } 2382 2383 pmap_unmap_ptes(pmap); 2384 /* 2385 * Now, if we've fallen through down to here, chances are that there 2386 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2387 */ 2388 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2389 u_int cnt; 2390 2391 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2392 if (pmap_active) { 2393 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2394 NBPG); 2395 *cleanlist[cnt].pte = 0; 2396 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2397 } else 2398 *cleanlist[cnt].pte = 0; 2399 pmap_pte_delref(pmap, cleanlist[cnt].va); 2400 } 2401 } 2402 PMAP_MAP_TO_HEAD_UNLOCK(); 2403 } 2404 2405 /* 2406 * Routine: pmap_remove_all 2407 * Function: 2408 * Removes this physical page from 2409 * all physical maps in which it resides. 2410 * Reflects back modify bits to the pager. 2411 */ 2412 2413 static void 2414 pmap_remove_all(pg) 2415 struct vm_page *pg; 2416 { 2417 struct pv_entry *pv, *npv; 2418 struct pmap *pmap; 2419 pt_entry_t *pte, *ptes; 2420 2421 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2422 2423 /* set vm_page => pmap locking */ 2424 PMAP_HEAD_TO_MAP_LOCK(); 2425 2426 simple_lock(&pg->mdpage.pvh_slock); 2427 2428 pv = pg->mdpage.pvh_list; 2429 if (pv == NULL) { 2430 PDEBUG(0, printf("free page\n")); 2431 simple_unlock(&pg->mdpage.pvh_slock); 2432 PMAP_HEAD_TO_MAP_UNLOCK(); 2433 return; 2434 } 2435 pmap_clean_page(pv, FALSE); 2436 2437 while (pv) { 2438 pmap = pv->pv_pmap; 2439 ptes = pmap_map_ptes(pmap); 2440 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2441 2442 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2443 pv->pv_va, pv->pv_flags)); 2444 #ifdef DEBUG 2445 if (!pmap_pde_page(pmap_pde(pmap, pv->pv_va)) || 2446 !pmap_pte_v(pte) || pmap_pte_pa(pte) != pa) 2447 panic("pmap_remove_all: bad mapping"); 2448 #endif /* DEBUG */ 2449 2450 /* 2451 * Update statistics 2452 */ 2453 --pmap->pm_stats.resident_count; 2454 2455 /* Wired bit */ 2456 if (pv->pv_flags & PT_W) 2457 --pmap->pm_stats.wired_count; 2458 2459 /* 2460 * Invalidate the PTEs. 2461 * XXX: should cluster them up and invalidate as many 2462 * as possible at once. 2463 */ 2464 2465 #ifdef needednotdone 2466 reduce wiring count on page table pages as references drop 2467 #endif 2468 2469 *pte = 0; 2470 pmap_pte_delref(pmap, pv->pv_va); 2471 2472 npv = pv->pv_next; 2473 pmap_free_pv(pmap, pv); 2474 pv = npv; 2475 pmap_unmap_ptes(pmap); 2476 } 2477 pg->mdpage.pvh_list = NULL; 2478 simple_unlock(&pg->mdpage.pvh_slock); 2479 PMAP_HEAD_TO_MAP_UNLOCK(); 2480 2481 PDEBUG(0, printf("done\n")); 2482 cpu_tlb_flushID(); 2483 cpu_cpwait(); 2484 } 2485 2486 2487 /* 2488 * Set the physical protection on the specified range of this map as requested. 2489 */ 2490 2491 void 2492 pmap_protect(pmap, sva, eva, prot) 2493 struct pmap *pmap; 2494 vaddr_t sva; 2495 vaddr_t eva; 2496 vm_prot_t prot; 2497 { 2498 pt_entry_t *pte = NULL, *ptes; 2499 struct vm_page *pg; 2500 int armprot; 2501 int flush = 0; 2502 paddr_t pa; 2503 2504 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2505 pmap, sva, eva, prot)); 2506 2507 if (~prot & VM_PROT_READ) { 2508 /* Just remove the mappings. */ 2509 pmap_remove(pmap, sva, eva); 2510 /* pmap_update not needed as it should be called by the caller 2511 * of pmap_protect */ 2512 return; 2513 } 2514 if (prot & VM_PROT_WRITE) { 2515 /* 2516 * If this is a read->write transition, just ignore it and let 2517 * uvm_fault() take care of it later. 2518 */ 2519 return; 2520 } 2521 2522 sva &= PG_FRAME; 2523 eva &= PG_FRAME; 2524 2525 /* Need to lock map->head */ 2526 PMAP_MAP_TO_HEAD_LOCK(); 2527 2528 ptes = pmap_map_ptes(pmap); 2529 /* 2530 * We need to acquire a pointer to a page table page before entering 2531 * the following loop. 2532 */ 2533 while (sva < eva) { 2534 if (pmap_pde_page(pmap_pde(pmap, sva))) 2535 break; 2536 sva = (sva & PD_MASK) + NBPD; 2537 } 2538 2539 pte = &ptes[arm_byte_to_page(sva)]; 2540 2541 while (sva < eva) { 2542 /* only check once in a while */ 2543 if ((sva & PT_MASK) == 0) { 2544 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2545 /* We can race ahead here, to the next pde. */ 2546 sva += NBPD; 2547 pte += arm_byte_to_page(NBPD); 2548 continue; 2549 } 2550 } 2551 2552 if (!pmap_pte_v(pte)) 2553 goto next; 2554 2555 flush = 1; 2556 2557 armprot = 0; 2558 if (sva < VM_MAXUSER_ADDRESS) 2559 armprot |= PT_AP(AP_U); 2560 else if (sva < VM_MAX_ADDRESS) 2561 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2562 *pte = (*pte & 0xfffff00f) | armprot; 2563 2564 pa = pmap_pte_pa(pte); 2565 2566 /* Get the physical page index */ 2567 2568 /* Clear write flag */ 2569 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2570 simple_lock(&pg->mdpage.pvh_slock); 2571 (void) pmap_modify_pv(pmap, sva, pg, PT_Wr, 0); 2572 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2573 simple_unlock(&pg->mdpage.pvh_slock); 2574 } 2575 2576 next: 2577 sva += NBPG; 2578 pte++; 2579 } 2580 pmap_unmap_ptes(pmap); 2581 PMAP_MAP_TO_HEAD_UNLOCK(); 2582 if (flush) 2583 cpu_tlb_flushID(); 2584 } 2585 2586 /* 2587 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2588 * int flags) 2589 * 2590 * Insert the given physical page (p) at 2591 * the specified virtual address (v) in the 2592 * target physical map with the protection requested. 2593 * 2594 * If specified, the page will be wired down, meaning 2595 * that the related pte can not be reclaimed. 2596 * 2597 * NB: This is the only routine which MAY NOT lazy-evaluate 2598 * or lose information. That is, this routine must actually 2599 * insert this page into the given map NOW. 2600 */ 2601 2602 int 2603 pmap_enter(pmap, va, pa, prot, flags) 2604 struct pmap *pmap; 2605 vaddr_t va; 2606 paddr_t pa; 2607 vm_prot_t prot; 2608 int flags; 2609 { 2610 pt_entry_t *pte, *ptes; 2611 u_int npte; 2612 paddr_t opa; 2613 int nflags; 2614 boolean_t wired = (flags & PMAP_WIRED) != 0; 2615 struct vm_page *pg; 2616 struct pv_entry *pve; 2617 int error; 2618 2619 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2620 va, pa, pmap, prot, wired)); 2621 2622 #ifdef DIAGNOSTIC 2623 /* Valid address ? */ 2624 if (va >= (pmap_curmaxkvaddr)) 2625 panic("pmap_enter: too big"); 2626 if (pmap != pmap_kernel() && va != 0) { 2627 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2628 panic("pmap_enter: kernel page in user map"); 2629 } else { 2630 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2631 panic("pmap_enter: user page in kernel map"); 2632 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2633 panic("pmap_enter: entering PT page"); 2634 } 2635 #endif 2636 /* 2637 * Get a pointer to the page. Later on in this function, we 2638 * test for a managed page by checking pg != NULL. 2639 */ 2640 pg = PHYS_TO_VM_PAGE(pa); 2641 2642 /* get lock */ 2643 PMAP_MAP_TO_HEAD_LOCK(); 2644 /* 2645 * Get a pointer to the pte for this virtual address. If the 2646 * pte pointer is NULL then we are missing the L2 page table 2647 * so we need to create one. 2648 */ 2649 /* XXX horrible hack to get us working with lockdebug */ 2650 simple_lock(&pmap->pm_obj.vmobjlock); 2651 pte = pmap_pte(pmap, va); 2652 if (!pte) { 2653 struct vm_page *ptp; 2654 KASSERT(pmap != pmap_kernel()); /* kernel should have pre-grown */ 2655 2656 /* if failure is allowed then don't try too hard */ 2657 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2658 if (ptp == NULL) { 2659 if (flags & PMAP_CANFAIL) { 2660 error = ENOMEM; 2661 goto out; 2662 } 2663 panic("pmap_enter: get ptp failed"); 2664 } 2665 2666 pte = pmap_pte(pmap, va); 2667 #ifdef DIAGNOSTIC 2668 if (!pte) 2669 panic("pmap_enter: no pte"); 2670 #endif 2671 } 2672 2673 nflags = 0; 2674 if (prot & VM_PROT_WRITE) 2675 nflags |= PT_Wr; 2676 if (wired) 2677 nflags |= PT_W; 2678 2679 /* More debugging info */ 2680 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2681 *pte)); 2682 2683 /* Is the pte valid ? If so then this page is already mapped */ 2684 if (pmap_pte_v(pte)) { 2685 /* Get the physical address of the current page mapped */ 2686 opa = pmap_pte_pa(pte); 2687 2688 #ifdef MYCROFT_HACK 2689 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2690 #endif 2691 2692 /* Are we mapping the same page ? */ 2693 if (opa == pa) { 2694 /* All we must be doing is changing the protection */ 2695 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2696 va, pa)); 2697 2698 /* Has the wiring changed ? */ 2699 if (pg != NULL) { 2700 simple_lock(&pg->mdpage.pvh_slock); 2701 (void) pmap_modify_pv(pmap, va, pg, 2702 PT_Wr | PT_W, nflags); 2703 simple_unlock(&pg->mdpage.pvh_slock); 2704 } 2705 } else { 2706 struct vm_page *opg; 2707 2708 /* We are replacing the page with a new one. */ 2709 cpu_idcache_wbinv_range(va, NBPG); 2710 2711 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2712 va, pa, opa)); 2713 2714 /* 2715 * If it is part of our managed memory then we 2716 * must remove it from the PV list 2717 */ 2718 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2719 simple_lock(&opg->mdpage.pvh_slock); 2720 pve = pmap_remove_pv(opg, pmap, va); 2721 simple_unlock(&opg->mdpage.pvh_slock); 2722 } else { 2723 pve = NULL; 2724 } 2725 2726 goto enter; 2727 } 2728 } else { 2729 opa = 0; 2730 pve = NULL; 2731 pmap_pte_addref(pmap, va); 2732 2733 /* pte is not valid so we must be hooking in a new page */ 2734 ++pmap->pm_stats.resident_count; 2735 2736 enter: 2737 /* 2738 * Enter on the PV list if part of our managed memory 2739 */ 2740 if (pmap_initialized && pg != NULL) { 2741 if (pve == NULL) { 2742 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2743 if (pve == NULL) { 2744 if (flags & PMAP_CANFAIL) { 2745 error = ENOMEM; 2746 goto out; 2747 } 2748 panic("pmap_enter: no pv entries available"); 2749 } 2750 } 2751 /* enter_pv locks pvh when adding */ 2752 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2753 } else { 2754 pg = NULL; 2755 if (pve != NULL) 2756 pmap_free_pv(pmap, pve); 2757 } 2758 } 2759 2760 #ifdef MYCROFT_HACK 2761 if (mycroft_hack) 2762 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2763 #endif 2764 2765 /* Construct the pte, giving the correct access. */ 2766 npte = (pa & PG_FRAME); 2767 2768 /* VA 0 is magic. */ 2769 if (pmap != pmap_kernel() && va != 0) 2770 npte |= PT_AP(AP_U); 2771 2772 if (pmap_initialized && pg != NULL) { 2773 #ifdef DIAGNOSTIC 2774 if ((flags & VM_PROT_ALL) & ~prot) 2775 panic("pmap_enter: access_type exceeds prot"); 2776 #endif 2777 npte |= pte_cache_mode; 2778 if (flags & VM_PROT_WRITE) { 2779 npte |= L2_SPAGE | PT_AP(AP_W); 2780 pg->mdpage.pvh_attrs |= PT_H | PT_M; 2781 } else if (flags & VM_PROT_ALL) { 2782 npte |= L2_SPAGE; 2783 pg->mdpage.pvh_attrs |= PT_H; 2784 } else 2785 npte |= L2_INVAL; 2786 } else { 2787 if (prot & VM_PROT_WRITE) 2788 npte |= L2_SPAGE | PT_AP(AP_W); 2789 else if (prot & VM_PROT_ALL) 2790 npte |= L2_SPAGE; 2791 else 2792 npte |= L2_INVAL; 2793 } 2794 2795 #ifdef MYCROFT_HACK 2796 if (mycroft_hack) 2797 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2798 #endif 2799 2800 *pte = npte; 2801 2802 if (pmap_initialized && pg != NULL) { 2803 boolean_t pmap_active = FALSE; 2804 /* XXX this will change once the whole of pmap_enter uses 2805 * map_ptes 2806 */ 2807 ptes = pmap_map_ptes(pmap); 2808 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2809 || (pmap == pmap_kernel())) 2810 pmap_active = TRUE; 2811 simple_lock(&pg->mdpage.pvh_slock); 2812 pmap_vac_me_harder(pmap, pg, ptes, pmap_active); 2813 simple_unlock(&pg->mdpage.pvh_slock); 2814 pmap_unmap_ptes(pmap); 2815 } 2816 2817 /* Better flush the TLB ... */ 2818 cpu_tlb_flushID_SE(va); 2819 error = 0; 2820 out: 2821 simple_unlock(&pmap->pm_obj.vmobjlock); 2822 PMAP_MAP_TO_HEAD_UNLOCK(); 2823 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2824 2825 return error; 2826 } 2827 2828 /* 2829 * pmap_kenter_pa: enter a kernel mapping 2830 * 2831 * => no need to lock anything assume va is already allocated 2832 * => should be faster than normal pmap enter function 2833 */ 2834 void 2835 pmap_kenter_pa(va, pa, prot) 2836 vaddr_t va; 2837 paddr_t pa; 2838 vm_prot_t prot; 2839 { 2840 pt_entry_t *pte; 2841 2842 pte = vtopte(va); 2843 KASSERT(!pmap_pte_v(pte)); 2844 *pte = L2_PTE(pa, AP_KRW); 2845 } 2846 2847 void 2848 pmap_kremove(va, len) 2849 vaddr_t va; 2850 vsize_t len; 2851 { 2852 pt_entry_t *pte; 2853 2854 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2855 2856 /* 2857 * We assume that we will only be called with small 2858 * regions of memory. 2859 */ 2860 2861 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2862 pte = vtopte(va); 2863 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2864 *pte = 0; 2865 cpu_tlb_flushID_SE(va); 2866 } 2867 } 2868 2869 /* 2870 * pmap_page_protect: 2871 * 2872 * Lower the permission for all mappings to a given page. 2873 */ 2874 2875 void 2876 pmap_page_protect(pg, prot) 2877 struct vm_page *pg; 2878 vm_prot_t prot; 2879 { 2880 2881 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2882 VM_PAGE_TO_PHYS(pg), prot)); 2883 2884 switch(prot) { 2885 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2886 case VM_PROT_READ|VM_PROT_WRITE: 2887 return; 2888 2889 case VM_PROT_READ: 2890 case VM_PROT_READ|VM_PROT_EXECUTE: 2891 pmap_copy_on_write(pg); 2892 break; 2893 2894 default: 2895 pmap_remove_all(pg); 2896 break; 2897 } 2898 } 2899 2900 2901 /* 2902 * Routine: pmap_unwire 2903 * Function: Clear the wired attribute for a map/virtual-address 2904 * pair. 2905 * In/out conditions: 2906 * The mapping must already exist in the pmap. 2907 */ 2908 2909 void 2910 pmap_unwire(pmap, va) 2911 struct pmap *pmap; 2912 vaddr_t va; 2913 { 2914 pt_entry_t *pte; 2915 paddr_t pa; 2916 struct vm_page *pg; 2917 2918 /* 2919 * Make sure pmap is valid. -dct 2920 */ 2921 if (pmap == NULL) 2922 return; 2923 2924 /* Get the pte */ 2925 pte = pmap_pte(pmap, va); 2926 if (!pte) 2927 return; 2928 2929 /* Extract the physical address of the page */ 2930 pa = pmap_pte_pa(pte); 2931 2932 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2933 return; 2934 2935 simple_lock(&pg->mdpage.pvh_slock); 2936 /* Update the wired bit in the pv entry for this page. */ 2937 (void) pmap_modify_pv(pmap, va, pg, PT_W, 0); 2938 simple_unlock(&pg->mdpage.pvh_slock); 2939 } 2940 2941 /* 2942 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 2943 * 2944 * Return the pointer to a page table entry corresponding to the supplied 2945 * virtual address. 2946 * 2947 * The page directory is first checked to make sure that a page table 2948 * for the address in question exists and if it does a pointer to the 2949 * entry is returned. 2950 * 2951 * The way this works is that that the kernel page tables are mapped 2952 * into the memory map at ALT_PAGE_TBLS_BASE to ALT_PAGE_TBLS_BASE+4MB. 2953 * This allows page tables to be located quickly. 2954 */ 2955 pt_entry_t * 2956 pmap_pte(pmap, va) 2957 struct pmap *pmap; 2958 vaddr_t va; 2959 { 2960 pt_entry_t *ptp; 2961 pt_entry_t *result; 2962 2963 /* The pmap must be valid */ 2964 if (!pmap) 2965 return(NULL); 2966 2967 /* Return the address of the pte */ 2968 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 2969 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 2970 2971 /* Do we have a valid pde ? If not we don't have a page table */ 2972 if (!pmap_pde_page(pmap_pde(pmap, va))) { 2973 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 2974 pmap_pde(pmap, va))); 2975 return(NULL); 2976 } 2977 2978 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 2979 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2980 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2981 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME))); 2982 2983 /* 2984 * If the pmap is the kernel pmap or the pmap is the active one 2985 * then we can just return a pointer to entry relative to 2986 * PROCESS_PAGE_TBLS_BASE. 2987 * Otherwise we need to map the page tables to an alternative 2988 * address and reference them there. 2989 */ 2990 if (pmap == pmap_kernel() || pmap->pm_pptpt 2991 == (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2992 + ((PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) & 2993 ~3) + (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 2994 ptp = (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 2995 } else { 2996 struct proc *p = curproc; 2997 2998 /* If we don't have a valid curproc use proc0 */ 2999 /* Perhaps we should just use kernel_pmap instead */ 3000 if (p == NULL) 3001 p = &proc0; 3002 #ifdef DIAGNOSTIC 3003 /* 3004 * The pmap should always be valid for the process so 3005 * panic if it is not. 3006 */ 3007 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 3008 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 3009 va, p, p->p_vmspace); 3010 console_debugger(); 3011 } 3012 /* 3013 * The pmap for the current process should be mapped. If it 3014 * is not then we have a problem. 3015 */ 3016 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 3017 (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3018 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3019 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 3020 printf("pmap pagetable = P%08lx current = P%08x ", 3021 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3022 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3023 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & 3024 PG_FRAME)); 3025 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 3026 panic("pmap_pte: current and pmap mismatch\n"); 3027 } 3028 #endif 3029 3030 ptp = (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3031 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3032 pmap->pm_pptpt, FALSE); 3033 cpu_tlb_flushD(); 3034 cpu_cpwait(); 3035 } 3036 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 3037 ((va >> (PGSHIFT-2)) & ~3))); 3038 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 3039 return(result); 3040 } 3041 3042 /* 3043 * Routine: pmap_extract 3044 * Function: 3045 * Extract the physical page address associated 3046 * with the given map/virtual_address pair. 3047 */ 3048 boolean_t 3049 pmap_extract(pmap, va, pap) 3050 struct pmap *pmap; 3051 vaddr_t va; 3052 paddr_t *pap; 3053 { 3054 pd_entry_t *pde; 3055 pt_entry_t *pte, *ptes; 3056 paddr_t pa; 3057 boolean_t rv = TRUE; 3058 3059 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 3060 3061 /* 3062 * Get the pte for this virtual address. 3063 */ 3064 pde = pmap_pde(pmap, va); 3065 ptes = pmap_map_ptes(pmap); 3066 pte = &ptes[arm_byte_to_page(va)]; 3067 3068 if (pmap_pde_section(pde)) { 3069 pa = (*pde & PD_MASK) | (va & (L1_SEC_SIZE - 1)); 3070 goto out; 3071 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3072 rv = FALSE; 3073 goto out; 3074 } 3075 3076 if ((*pte & L2_MASK) == L2_LPAGE) { 3077 /* Extract the physical address from the pte */ 3078 pa = *pte & ~(L2_LPAGE_SIZE - 1); 3079 3080 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3081 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3082 3083 if (pap != NULL) 3084 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3085 goto out; 3086 } 3087 3088 /* Extract the physical address from the pte */ 3089 pa = pmap_pte_pa(pte); 3090 3091 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3092 (pa | (va & ~PG_FRAME)))); 3093 3094 if (pap != NULL) 3095 *pap = pa | (va & ~PG_FRAME); 3096 out: 3097 pmap_unmap_ptes(pmap); 3098 return (rv); 3099 } 3100 3101 3102 /* 3103 * Copy the range specified by src_addr/len from the source map to the 3104 * range dst_addr/len in the destination map. 3105 * 3106 * This routine is only advisory and need not do anything. 3107 */ 3108 3109 void 3110 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3111 struct pmap *dst_pmap; 3112 struct pmap *src_pmap; 3113 vaddr_t dst_addr; 3114 vsize_t len; 3115 vaddr_t src_addr; 3116 { 3117 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3118 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3119 } 3120 3121 #if defined(PMAP_DEBUG) 3122 void 3123 pmap_dump_pvlist(phys, m) 3124 vaddr_t phys; 3125 char *m; 3126 { 3127 struct vm_page *pg; 3128 struct pv_entry *pv; 3129 3130 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3131 printf("INVALID PA\n"); 3132 return; 3133 } 3134 simple_lock(&pg->mdpage.pvh_slock); 3135 printf("%s %08lx:", m, phys); 3136 if (pg->mdpage.pvh_list == NULL) { 3137 printf(" no mappings\n"); 3138 return; 3139 } 3140 3141 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3142 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3143 pv->pv_va, pv->pv_flags); 3144 3145 printf("\n"); 3146 simple_unlock(&pg->mdpage.pvh_slock); 3147 } 3148 3149 #endif /* PMAP_DEBUG */ 3150 3151 __inline static boolean_t 3152 pmap_testbit(pg, setbits) 3153 struct vm_page *pg; 3154 unsigned int setbits; 3155 { 3156 3157 PDEBUG(1, printf("pmap_testbit: pa=%08lx set=%08x\n", 3158 VM_PAGE_TO_PHYS(pg), setbits)); 3159 3160 /* 3161 * Check saved info only 3162 */ 3163 if (pg->mdpage.pvh_attrs & setbits) { 3164 PDEBUG(0, printf("pmap_attributes = %02x\n", 3165 pg->mdpage.pvh_attrs)); 3166 return(TRUE); 3167 } 3168 3169 return(FALSE); 3170 } 3171 3172 static pt_entry_t * 3173 pmap_map_ptes(struct pmap *pmap) 3174 { 3175 struct proc *p; 3176 3177 /* the kernel's pmap is always accessible */ 3178 if (pmap == pmap_kernel()) { 3179 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE ; 3180 } 3181 3182 if (pmap_is_curpmap(pmap)) { 3183 simple_lock(&pmap->pm_obj.vmobjlock); 3184 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3185 } 3186 3187 p = curproc; 3188 3189 if (p == NULL) 3190 p = &proc0; 3191 3192 /* need to lock both curpmap and pmap: use ordered locking */ 3193 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3194 simple_lock(&pmap->pm_obj.vmobjlock); 3195 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3196 } else { 3197 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3198 simple_lock(&pmap->pm_obj.vmobjlock); 3199 } 3200 3201 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3202 pmap->pm_pptpt, FALSE); 3203 cpu_tlb_flushD(); 3204 cpu_cpwait(); 3205 return (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3206 } 3207 3208 /* 3209 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3210 */ 3211 3212 static void 3213 pmap_unmap_ptes(pmap) 3214 struct pmap *pmap; 3215 { 3216 if (pmap == pmap_kernel()) { 3217 return; 3218 } 3219 if (pmap_is_curpmap(pmap)) { 3220 simple_unlock(&pmap->pm_obj.vmobjlock); 3221 } else { 3222 simple_unlock(&pmap->pm_obj.vmobjlock); 3223 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3224 } 3225 } 3226 3227 /* 3228 * Modify pte bits for all ptes corresponding to the given physical address. 3229 * We use `maskbits' rather than `clearbits' because we're always passing 3230 * constants and the latter would require an extra inversion at run-time. 3231 */ 3232 3233 static void 3234 pmap_clearbit(pg, maskbits) 3235 struct vm_page *pg; 3236 unsigned int maskbits; 3237 { 3238 struct pv_entry *pv; 3239 pt_entry_t *pte; 3240 vaddr_t va; 3241 int tlbentry; 3242 3243 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3244 VM_PAGE_TO_PHYS(pg), maskbits)); 3245 3246 tlbentry = 0; 3247 3248 PMAP_HEAD_TO_MAP_LOCK(); 3249 simple_lock(&pg->mdpage.pvh_slock); 3250 3251 /* 3252 * Clear saved attributes (modify, reference) 3253 */ 3254 pg->mdpage.pvh_attrs &= ~maskbits; 3255 3256 if (pg->mdpage.pvh_list == NULL) { 3257 simple_unlock(&pg->mdpage.pvh_slock); 3258 PMAP_HEAD_TO_MAP_UNLOCK(); 3259 return; 3260 } 3261 3262 /* 3263 * Loop over all current mappings setting/clearing as appropos 3264 */ 3265 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3266 va = pv->pv_va; 3267 pv->pv_flags &= ~maskbits; 3268 pte = pmap_pte(pv->pv_pmap, va); 3269 KASSERT(pte != NULL); 3270 if (maskbits & (PT_Wr|PT_M)) { 3271 if ((pv->pv_flags & PT_NC)) { 3272 /* 3273 * Entry is not cacheable: reenable 3274 * the cache, nothing to flush 3275 * 3276 * Don't turn caching on again if this 3277 * is a modified emulation. This 3278 * would be inconsitent with the 3279 * settings created by 3280 * pmap_vac_me_harder(). 3281 * 3282 * There's no need to call 3283 * pmap_vac_me_harder() here: all 3284 * pages are loosing their write 3285 * permission. 3286 * 3287 */ 3288 if (maskbits & PT_Wr) { 3289 *pte |= pte_cache_mode; 3290 pv->pv_flags &= ~PT_NC; 3291 } 3292 } else if (pmap_is_curpmap(pv->pv_pmap)) 3293 /* 3294 * Entry is cacheable: check if pmap is 3295 * current if it is flush it, 3296 * otherwise it won't be in the cache 3297 */ 3298 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3299 3300 /* make the pte read only */ 3301 *pte &= ~PT_AP(AP_W); 3302 } 3303 3304 if (maskbits & PT_H) 3305 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3306 3307 if (pmap_is_curpmap(pv->pv_pmap)) 3308 /* 3309 * if we had cacheable pte's we'd clean the 3310 * pte out to memory here 3311 * 3312 * flush tlb entry as it's in the current pmap 3313 */ 3314 cpu_tlb_flushID_SE(pv->pv_va); 3315 } 3316 cpu_cpwait(); 3317 3318 simple_unlock(&pg->mdpage.pvh_slock); 3319 PMAP_HEAD_TO_MAP_UNLOCK(); 3320 } 3321 3322 3323 boolean_t 3324 pmap_clear_modify(pg) 3325 struct vm_page *pg; 3326 { 3327 boolean_t rv; 3328 3329 PDEBUG(0, printf("pmap_clear_modify pa=%08lx\n", VM_PAGE_TO_PHYS(pg))); 3330 rv = pmap_testbit(pg, PT_M); 3331 pmap_clearbit(pg, PT_M); 3332 return rv; 3333 } 3334 3335 3336 boolean_t 3337 pmap_clear_reference(pg) 3338 struct vm_page *pg; 3339 { 3340 boolean_t rv; 3341 3342 PDEBUG(0, printf("pmap_clear_reference pa=%08lx\n", 3343 VM_PAGE_TO_PHYS(pg))); 3344 rv = pmap_testbit(pg, PT_H); 3345 pmap_clearbit(pg, PT_H); 3346 return rv; 3347 } 3348 3349 3350 void 3351 pmap_copy_on_write(pg) 3352 struct vm_page *pg; 3353 { 3354 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", VM_PAGE_TO_PHYS(pg))); 3355 pmap_clearbit(pg, PT_Wr); 3356 } 3357 3358 3359 boolean_t 3360 pmap_is_modified(pg) 3361 struct vm_page *pg; 3362 { 3363 boolean_t result; 3364 3365 result = pmap_testbit(pg, PT_M); 3366 PDEBUG(1, printf("pmap_is_modified pa=%08lx %x\n", 3367 VM_PAGE_TO_PHYS(pg), result)); 3368 return (result); 3369 } 3370 3371 3372 boolean_t 3373 pmap_is_referenced(pg) 3374 struct vm_page *pg; 3375 { 3376 boolean_t result; 3377 3378 result = pmap_testbit(pg, PT_H); 3379 PDEBUG(0, printf("pmap_is_referenced pa=%08lx %x\n", 3380 VM_PAGE_TO_PHYS(pg), result)); 3381 return (result); 3382 } 3383 3384 3385 int 3386 pmap_modified_emulation(pmap, va) 3387 struct pmap *pmap; 3388 vaddr_t va; 3389 { 3390 pt_entry_t *pte; 3391 paddr_t pa; 3392 struct vm_page *pg; 3393 u_int flags; 3394 3395 PDEBUG(2, printf("pmap_modified_emulation\n")); 3396 3397 /* Get the pte */ 3398 pte = pmap_pte(pmap, va); 3399 if (!pte) { 3400 PDEBUG(2, printf("no pte\n")); 3401 return(0); 3402 } 3403 3404 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3405 3406 /* Check for a zero pte */ 3407 if (*pte == 0) 3408 return(0); 3409 3410 /* This can happen if user code tries to access kernel memory. */ 3411 if ((*pte & PT_AP(AP_W)) != 0) 3412 return (0); 3413 3414 /* Extract the physical address of the page */ 3415 pa = pmap_pte_pa(pte); 3416 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3417 return(0); 3418 3419 /* Get the current flags for this page. */ 3420 PMAP_HEAD_TO_MAP_LOCK(); 3421 simple_lock(&pg->mdpage.pvh_slock); 3422 3423 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3424 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3425 3426 /* 3427 * Do the flags say this page is writable ? If not then it is a 3428 * genuine write fault. If yes then the write fault is our fault 3429 * as we did not reflect the write access in the PTE. Now we know 3430 * a write has occurred we can correct this and also set the 3431 * modified bit 3432 */ 3433 if (~flags & PT_Wr) { 3434 simple_unlock(&pg->mdpage.pvh_slock); 3435 PMAP_HEAD_TO_MAP_UNLOCK(); 3436 return(0); 3437 } 3438 3439 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3440 va, pte, *pte)); 3441 pg->mdpage.pvh_attrs |= PT_H | PT_M; 3442 3443 /* 3444 * Re-enable write permissions for the page. No need to call 3445 * pmap_vac_me_harder(), since this is just a 3446 * modified-emulation fault, and the PT_Wr bit isn't changing. We've 3447 * already set the cacheable bits based on the assumption that we 3448 * can write to this page. 3449 */ 3450 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3451 PDEBUG(0, printf("->(%08x)\n", *pte)); 3452 3453 simple_unlock(&pg->mdpage.pvh_slock); 3454 PMAP_HEAD_TO_MAP_UNLOCK(); 3455 /* Return, indicating the problem has been dealt with */ 3456 cpu_tlb_flushID_SE(va); 3457 cpu_cpwait(); 3458 return(1); 3459 } 3460 3461 3462 int 3463 pmap_handled_emulation(pmap, va) 3464 struct pmap *pmap; 3465 vaddr_t va; 3466 { 3467 pt_entry_t *pte; 3468 paddr_t pa; 3469 struct vm_page *pg; 3470 3471 PDEBUG(2, printf("pmap_handled_emulation\n")); 3472 3473 /* Get the pte */ 3474 pte = pmap_pte(pmap, va); 3475 if (!pte) { 3476 PDEBUG(2, printf("no pte\n")); 3477 return(0); 3478 } 3479 3480 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3481 3482 /* Check for a zero pte */ 3483 if (*pte == 0) 3484 return(0); 3485 3486 /* This can happen if user code tries to access kernel memory. */ 3487 if ((*pte & L2_MASK) != L2_INVAL) 3488 return (0); 3489 3490 /* Extract the physical address of the page */ 3491 pa = pmap_pte_pa(pte); 3492 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3493 return (0); 3494 3495 /* 3496 * Ok we just enable the pte and mark the attibs as handled 3497 */ 3498 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3499 va, pte, *pte)); 3500 pg->mdpage.pvh_attrs |= PT_H; 3501 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3502 PDEBUG(0, printf("->(%08x)\n", *pte)); 3503 3504 /* Return, indicating the problem has been dealt with */ 3505 cpu_tlb_flushID_SE(va); 3506 cpu_cpwait(); 3507 return(1); 3508 } 3509 3510 3511 3512 3513 /* 3514 * pmap_collect: free resources held by a pmap 3515 * 3516 * => optional function. 3517 * => called when a process is swapped out to free memory. 3518 */ 3519 3520 void 3521 pmap_collect(pmap) 3522 struct pmap *pmap; 3523 { 3524 } 3525 3526 /* 3527 * Routine: pmap_procwr 3528 * 3529 * Function: 3530 * Synchronize caches corresponding to [addr, addr+len) in p. 3531 * 3532 */ 3533 void 3534 pmap_procwr(p, va, len) 3535 struct proc *p; 3536 vaddr_t va; 3537 int len; 3538 { 3539 /* We only need to do anything if it is the current process. */ 3540 if (p == curproc) 3541 cpu_icache_sync_range(va, len); 3542 } 3543 /* 3544 * PTP functions 3545 */ 3546 3547 /* 3548 * pmap_steal_ptp: Steal a PTP from somewhere else. 3549 * 3550 * This is just a placeholder, for now we never steal. 3551 */ 3552 3553 static struct vm_page * 3554 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3555 { 3556 return (NULL); 3557 } 3558 3559 /* 3560 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3561 * 3562 * => pmap should NOT be pmap_kernel() 3563 * => pmap should be locked 3564 */ 3565 3566 static struct vm_page * 3567 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3568 { 3569 struct vm_page *ptp; 3570 3571 if (pmap_pde_page(pmap_pde(pmap, va))) { 3572 3573 /* valid... check hint (saves us a PA->PG lookup) */ 3574 #if 0 3575 if (pmap->pm_ptphint && 3576 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3577 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3578 return (pmap->pm_ptphint); 3579 #endif 3580 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3581 #ifdef DIAGNOSTIC 3582 if (ptp == NULL) 3583 panic("pmap_get_ptp: unmanaged user PTP"); 3584 #endif 3585 // pmap->pm_ptphint = ptp; 3586 return(ptp); 3587 } 3588 3589 /* allocate a new PTP (updates ptphint) */ 3590 return(pmap_alloc_ptp(pmap, va, just_try)); 3591 } 3592 3593 /* 3594 * pmap_alloc_ptp: allocate a PTP for a PMAP 3595 * 3596 * => pmap should already be locked by caller 3597 * => we use the ptp's wire_count to count the number of active mappings 3598 * in the PTP (we start it at one to prevent any chance this PTP 3599 * will ever leak onto the active/inactive queues) 3600 */ 3601 3602 /*__inline */ static struct vm_page * 3603 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3604 { 3605 struct vm_page *ptp; 3606 3607 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3608 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3609 if (ptp == NULL) { 3610 if (just_try) 3611 return (NULL); 3612 3613 ptp = pmap_steal_ptp(pmap, va); 3614 3615 if (ptp == NULL) 3616 return (NULL); 3617 /* Stole a page, zero it. */ 3618 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3619 } 3620 3621 /* got one! */ 3622 ptp->flags &= ~PG_BUSY; /* never busy */ 3623 ptp->wire_count = 1; /* no mappings yet */ 3624 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3625 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3626 // pmap->pm_ptphint = ptp; 3627 return (ptp); 3628 } 3629 3630 vaddr_t 3631 pmap_growkernel(maxkvaddr) 3632 vaddr_t maxkvaddr; 3633 { 3634 struct pmap *kpm = pmap_kernel(), *pm; 3635 int s; 3636 paddr_t ptaddr; 3637 struct vm_page *ptp; 3638 3639 if (maxkvaddr <= pmap_curmaxkvaddr) 3640 goto out; /* we are OK */ 3641 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3642 pmap_curmaxkvaddr, maxkvaddr)); 3643 3644 /* 3645 * whoops! we need to add kernel PTPs 3646 */ 3647 3648 s = splhigh(); /* to be safe */ 3649 simple_lock(&kpm->pm_obj.vmobjlock); 3650 /* due to the way the arm pmap works we map 4MB at a time */ 3651 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr ; pmap_curmaxkvaddr += 4 * NBPD) { 3652 3653 if (uvm.page_init_done == FALSE) { 3654 3655 /* 3656 * we're growing the kernel pmap early (from 3657 * uvm_pageboot_alloc()). this case must be 3658 * handled a little differently. 3659 */ 3660 3661 if (uvm_page_physget(&ptaddr) == FALSE) 3662 panic("pmap_growkernel: out of memory"); 3663 pmap_zero_page(ptaddr); 3664 3665 /* map this page in */ 3666 pmap_map_in_l1(kpm, (pmap_curmaxkvaddr + 1), ptaddr, TRUE); 3667 3668 /* count PTP as resident */ 3669 kpm->pm_stats.resident_count++; 3670 continue; 3671 } 3672 3673 /* 3674 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3675 * pmap_initialized == FALSE CASE! WE MAY BE 3676 * INVOKED WHILE pmap_init() IS RUNNING! 3677 */ 3678 3679 if ((ptp = pmap_alloc_ptp(kpm, (pmap_curmaxkvaddr + 1), FALSE)) == NULL) { 3680 panic("pmap_growkernel: alloc ptp failed"); 3681 } 3682 3683 /* distribute new kernel PTP to all active pmaps */ 3684 simple_lock(&pmaps_lock); 3685 LIST_FOREACH(pm, &pmaps, pm_list) { 3686 pmap_map_in_l1(pm, (pmap_curmaxkvaddr + 1), VM_PAGE_TO_PHYS(ptp), TRUE); 3687 } 3688 3689 simple_unlock(&pmaps_lock); 3690 } 3691 3692 /* 3693 * flush out the cache, expensive but growkernel will happen so 3694 * rarely 3695 */ 3696 cpu_tlb_flushD(); 3697 cpu_cpwait(); 3698 3699 simple_unlock(&kpm->pm_obj.vmobjlock); 3700 splx(s); 3701 3702 out: 3703 return (pmap_curmaxkvaddr); 3704 } 3705 3706 3707 3708 /************************ Bootstrapping routines ****************************/ 3709 3710 /* 3711 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3712 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3713 * find them as necessary. 3714 * 3715 * Note that the data on this list is not valid after initarm() returns. 3716 */ 3717 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3718 3719 static vaddr_t 3720 kernel_pt_lookup(paddr_t pa) 3721 { 3722 pv_addr_t *pv; 3723 3724 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3725 if (pv->pv_pa == pa) 3726 return (pv->pv_va); 3727 } 3728 return (0); 3729 } 3730 3731 /* 3732 * pmap_map_section: 3733 * 3734 * Create a single section mapping. 3735 */ 3736 void 3737 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3738 { 3739 pd_entry_t *pde = (pd_entry_t *) l1pt; 3740 pd_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3741 pd_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3742 3743 KASSERT(((va | pa) & (L1_SEC_SIZE - 1)) == 0); 3744 3745 pde[va >> PDSHIFT] = L1_SECPTE(pa & PD_MASK, ap, fl); 3746 } 3747 3748 /* 3749 * pmap_map_entry: 3750 * 3751 * Create a single page mapping. 3752 */ 3753 void 3754 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3755 { 3756 pd_entry_t *pde = (pd_entry_t *) l1pt; 3757 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3758 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3759 pt_entry_t *pte; 3760 3761 KASSERT(((va | pa) & PGOFSET) == 0); 3762 3763 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3764 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3765 3766 pte = (pt_entry_t *) 3767 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3768 if (pte == NULL) 3769 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3770 3771 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa & PG_FRAME, ap, fl); 3772 } 3773 3774 /* 3775 * pmap_link_l2pt: 3776 * 3777 * Link the L2 page table specified by "pa" into the L1 3778 * page table at the slot for "va". 3779 */ 3780 void 3781 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3782 { 3783 pd_entry_t *pde = (pd_entry_t *) l1pt; 3784 u_int slot = va >> PDSHIFT; 3785 3786 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3787 3788 pde[slot + 0] = L1_PTE(l2pv->pv_pa + 0x000); 3789 pde[slot + 1] = L1_PTE(l2pv->pv_pa + 0x400); 3790 pde[slot + 2] = L1_PTE(l2pv->pv_pa + 0x800); 3791 pde[slot + 3] = L1_PTE(l2pv->pv_pa + 0xc00); 3792 3793 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3794 } 3795 3796 /* 3797 * pmap_map_chunk: 3798 * 3799 * Map a chunk of memory using the most efficient mappings 3800 * possible (section, large page, small page) into the 3801 * provided L1 and L2 tables at the specified virtual address. 3802 */ 3803 vsize_t 3804 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3805 int prot, int cache) 3806 { 3807 pd_entry_t *pde = (pd_entry_t *) l1pt; 3808 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3809 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3810 pt_entry_t *pte; 3811 vsize_t resid; 3812 int i; 3813 3814 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3815 3816 if (l1pt == 0) 3817 panic("pmap_map_chunk: no L1 table provided"); 3818 3819 #ifdef VERBOSE_INIT_ARM 3820 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3821 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3822 #endif 3823 3824 size = resid; 3825 3826 while (resid > 0) { 3827 /* See if we can use a section mapping. */ 3828 if (((pa | va) & (L1_SEC_SIZE - 1)) == 0 && 3829 resid >= L1_SEC_SIZE) { 3830 #ifdef VERBOSE_INIT_ARM 3831 printf("S"); 3832 #endif 3833 pde[va >> PDSHIFT] = L1_SECPTE(pa, ap, fl); 3834 va += L1_SEC_SIZE; 3835 pa += L1_SEC_SIZE; 3836 resid -= L1_SEC_SIZE; 3837 continue; 3838 } 3839 3840 /* 3841 * Ok, we're going to use an L2 table. Make sure 3842 * one is actually in the corresponding L1 slot 3843 * for the current VA. 3844 */ 3845 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3846 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3847 3848 pte = (pt_entry_t *) 3849 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3850 if (pte == NULL) 3851 panic("pmap_map_chunk: can't find L2 table for VA" 3852 "0x%08lx", va); 3853 3854 /* See if we can use a L2 large page mapping. */ 3855 if (((pa | va) & (L2_LPAGE_SIZE - 1)) == 0 && 3856 resid >= L2_LPAGE_SIZE) { 3857 #ifdef VERBOSE_INIT_ARM 3858 printf("L"); 3859 #endif 3860 for (i = 0; i < 16; i++) { 3861 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3862 L2_LPTE(pa, ap, fl); 3863 } 3864 va += L2_LPAGE_SIZE; 3865 pa += L2_LPAGE_SIZE; 3866 resid -= L2_LPAGE_SIZE; 3867 continue; 3868 } 3869 3870 /* Use a small page mapping. */ 3871 #ifdef VERBOSE_INIT_ARM 3872 printf("P"); 3873 #endif 3874 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa, ap, fl); 3875 va += NBPG; 3876 pa += NBPG; 3877 resid -= NBPG; 3878 } 3879 #ifdef VERBOSE_INIT_ARM 3880 printf("\n"); 3881 #endif 3882 return (size); 3883 } 3884
Indexes created Fri Sep 26 08:10:20 GMT 2025