pmap.c revision 1.50
1 /* $NetBSD: pmap.c,v 1.50 2002/03/05 04:48:03 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.50 2002/03/05 04:48:03 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static struct pmap_head pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 pagehook_t page_hook0; 199 pagehook_t page_hook1; 200 char *memhook; 201 pt_entry_t msgbufpte; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 static void pmap_free_l1pt __P((struct l1pt *)); 271 static int pmap_allocpagedir __P((struct pmap *)); 272 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 273 static void pmap_remove_all __P((struct vm_page *)); 274 275 276 vsize_t npages; 277 278 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t, boolean_t)); 279 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t, boolean_t)); 280 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 281 282 extern paddr_t physical_start; 283 extern paddr_t physical_freestart; 284 extern paddr_t physical_end; 285 extern paddr_t physical_freeend; 286 extern unsigned int free_pages; 287 extern int max_processes; 288 289 vaddr_t virtual_start; 290 vaddr_t virtual_end; 291 vaddr_t pmap_curmaxkvaddr; 292 293 vaddr_t avail_start; 294 vaddr_t avail_end; 295 296 extern pv_addr_t systempage; 297 298 #define ALLOC_PAGE_HOOK(x, s) \ 299 x.va = virtual_start; \ 300 x.pte = (pt_entry_t *)pmap_pte(pmap_kernel(), virtual_start); \ 301 virtual_start += s; 302 303 /* Variables used by the L1 page table queue code */ 304 SIMPLEQ_HEAD(l1pt_queue, l1pt); 305 struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 306 int l1pt_static_queue_count; /* items in the static l1 queue */ 307 int l1pt_static_create_count; /* static l1 items created */ 308 struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 309 int l1pt_queue_count; /* items in the l1 queue */ 310 int l1pt_create_count; /* stat - L1's create count */ 311 int l1pt_reuse_count; /* stat - L1's reused count */ 312 313 /* Local function prototypes (not used outside this file) */ 314 pt_entry_t *pmap_pte __P((struct pmap *pmap, vaddr_t va)); 315 void pmap_copy_on_write __P((struct vm_page *)); 316 void pmap_pinit __P((struct pmap *)); 317 void pmap_freepagedir __P((struct pmap *)); 318 319 /* Other function prototypes */ 320 extern void bzero_page __P((vaddr_t)); 321 extern void bcopy_page __P((vaddr_t, vaddr_t)); 322 323 struct l1pt *pmap_alloc_l1pt __P((void)); 324 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 325 vaddr_t l2pa, boolean_t)); 326 327 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 328 static void pmap_unmap_ptes __P((struct pmap *)); 329 330 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 331 pt_entry_t *, boolean_t)); 332 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 333 pt_entry_t *, boolean_t)); 334 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 335 pt_entry_t *, boolean_t)); 336 337 /* 338 * Cache enable bits in PTE to use on pages that are cacheable. 339 * On most machines this is cacheable/bufferable, but on some, eg arm10, we 340 * can chose between write-through and write-back cacheing. 341 */ 342 pt_entry_t pte_cache_mode = (PT_C | PT_B); 343 344 /* 345 * real definition of pv_entry. 346 */ 347 348 struct pv_entry { 349 struct pv_entry *pv_next; /* next pv_entry */ 350 struct pmap *pv_pmap; /* pmap where mapping lies */ 351 vaddr_t pv_va; /* virtual address for mapping */ 352 int pv_flags; /* flags */ 353 struct vm_page *pv_ptp; /* vm_page for the ptp */ 354 }; 355 356 /* 357 * pv_entrys are dynamically allocated in chunks from a single page. 358 * we keep track of how many pv_entrys are in use for each page and 359 * we can free pv_entry pages if needed. there is one lock for the 360 * entire allocation system. 361 */ 362 363 struct pv_page_info { 364 TAILQ_ENTRY(pv_page) pvpi_list; 365 struct pv_entry *pvpi_pvfree; 366 int pvpi_nfree; 367 }; 368 369 /* 370 * number of pv_entry's in a pv_page 371 * (note: won't work on systems where NPBG isn't a constant) 372 */ 373 374 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 375 sizeof(struct pv_entry)) 376 377 /* 378 * a pv_page: where pv_entrys are allocated from 379 */ 380 381 struct pv_page { 382 struct pv_page_info pvinfo; 383 struct pv_entry pvents[PVE_PER_PVPAGE]; 384 }; 385 386 #ifdef MYCROFT_HACK 387 int mycroft_hack = 0; 388 #endif 389 390 /* Function to set the debug level of the pmap code */ 391 392 #ifdef PMAP_DEBUG 393 void 394 pmap_debug(level) 395 int level; 396 { 397 pmap_debug_level = level; 398 printf("pmap_debug: level=%d\n", pmap_debug_level); 399 } 400 #endif /* PMAP_DEBUG */ 401 402 __inline static boolean_t 403 pmap_is_curpmap(struct pmap *pmap) 404 { 405 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 406 || (pmap == pmap_kernel())) 407 return (TRUE); 408 return (FALSE); 409 } 410 #include "isadma.h" 411 412 #if NISADMA > 0 413 /* 414 * Used to protect memory for ISA DMA bounce buffers. If, when loading 415 * pages into the system, memory intersects with any of these ranges, 416 * the intersecting memory will be loaded into a lower-priority free list. 417 */ 418 bus_dma_segment_t *pmap_isa_dma_ranges; 419 int pmap_isa_dma_nranges; 420 421 boolean_t pmap_isa_dma_range_intersect __P((paddr_t, psize_t, 422 paddr_t *, psize_t *)); 423 424 /* 425 * Check if a memory range intersects with an ISA DMA range, and 426 * return the page-rounded intersection if it does. The intersection 427 * will be placed on a lower-priority free list. 428 */ 429 boolean_t 430 pmap_isa_dma_range_intersect(pa, size, pap, sizep) 431 paddr_t pa; 432 psize_t size; 433 paddr_t *pap; 434 psize_t *sizep; 435 { 436 bus_dma_segment_t *ds; 437 int i; 438 439 if (pmap_isa_dma_ranges == NULL) 440 return (FALSE); 441 442 for (i = 0, ds = pmap_isa_dma_ranges; 443 i < pmap_isa_dma_nranges; i++, ds++) { 444 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 445 /* 446 * Beginning of region intersects with this range. 447 */ 448 *pap = trunc_page(pa); 449 *sizep = round_page(min(pa + size, 450 ds->ds_addr + ds->ds_len) - pa); 451 return (TRUE); 452 } 453 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 454 /* 455 * End of region intersects with this range. 456 */ 457 *pap = trunc_page(ds->ds_addr); 458 *sizep = round_page(min((pa + size) - ds->ds_addr, 459 ds->ds_len)); 460 return (TRUE); 461 } 462 } 463 464 /* 465 * No intersection found. 466 */ 467 return (FALSE); 468 } 469 #endif /* NISADMA > 0 */ 470 471 /* 472 * p v _ e n t r y f u n c t i o n s 473 */ 474 475 /* 476 * pv_entry allocation functions: 477 * the main pv_entry allocation functions are: 478 * pmap_alloc_pv: allocate a pv_entry structure 479 * pmap_free_pv: free one pv_entry 480 * pmap_free_pvs: free a list of pv_entrys 481 * 482 * the rest are helper functions 483 */ 484 485 /* 486 * pmap_alloc_pv: inline function to allocate a pv_entry structure 487 * => we lock pvalloc_lock 488 * => if we fail, we call out to pmap_alloc_pvpage 489 * => 3 modes: 490 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 491 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 492 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 493 * one now 494 * 495 * "try" is for optional functions like pmap_copy(). 496 */ 497 498 __inline static struct pv_entry * 499 pmap_alloc_pv(pmap, mode) 500 struct pmap *pmap; 501 int mode; 502 { 503 struct pv_page *pvpage; 504 struct pv_entry *pv; 505 506 simple_lock(&pvalloc_lock); 507 508 if (pv_freepages.tqh_first != NULL) { 509 pvpage = pv_freepages.tqh_first; 510 pvpage->pvinfo.pvpi_nfree--; 511 if (pvpage->pvinfo.pvpi_nfree == 0) { 512 /* nothing left in this one? */ 513 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 514 } 515 pv = pvpage->pvinfo.pvpi_pvfree; 516 #ifdef DIAGNOSTIC 517 if (pv == NULL) 518 panic("pmap_alloc_pv: pvpi_nfree off"); 519 #endif 520 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 521 pv_nfpvents--; /* took one from pool */ 522 } else { 523 pv = NULL; /* need more of them */ 524 } 525 526 /* 527 * if below low water mark or we didn't get a pv_entry we try and 528 * create more pv_entrys ... 529 */ 530 531 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 532 if (pv == NULL) 533 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 534 mode : ALLOCPV_NEED); 535 else 536 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 537 } 538 539 simple_unlock(&pvalloc_lock); 540 return(pv); 541 } 542 543 /* 544 * pmap_alloc_pvpage: maybe allocate a new pvpage 545 * 546 * if need_entry is false: try and allocate a new pv_page 547 * if need_entry is true: try and allocate a new pv_page and return a 548 * new pv_entry from it. if we are unable to allocate a pv_page 549 * we make a last ditch effort to steal a pv_page from some other 550 * mapping. if that fails, we panic... 551 * 552 * => we assume that the caller holds pvalloc_lock 553 */ 554 555 static struct pv_entry * 556 pmap_alloc_pvpage(pmap, mode) 557 struct pmap *pmap; 558 int mode; 559 { 560 struct vm_page *pg; 561 struct pv_page *pvpage; 562 struct pv_entry *pv; 563 int s; 564 565 /* 566 * if we need_entry and we've got unused pv_pages, allocate from there 567 */ 568 569 if (mode != ALLOCPV_NONEED && pv_unusedpgs.tqh_first != NULL) { 570 571 /* move it to pv_freepages list */ 572 pvpage = pv_unusedpgs.tqh_first; 573 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 574 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 575 576 /* allocate a pv_entry */ 577 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 578 pv = pvpage->pvinfo.pvpi_pvfree; 579 #ifdef DIAGNOSTIC 580 if (pv == NULL) 581 panic("pmap_alloc_pvpage: pvpi_nfree off"); 582 #endif 583 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 584 585 pv_nfpvents--; /* took one from pool */ 586 return(pv); 587 } 588 589 /* 590 * see if we've got a cached unmapped VA that we can map a page in. 591 * if not, try to allocate one. 592 */ 593 594 595 if (pv_cachedva == 0) { 596 s = splvm(); 597 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 598 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 599 splx(s); 600 if (pv_cachedva == 0) { 601 return (NULL); 602 } 603 } 604 605 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 606 UVM_PGA_USERESERVE); 607 if (pg) 608 pg->flags &= ~PG_BUSY; /* never busy */ 609 610 if (pg == NULL) 611 return (NULL); 612 613 /* 614 * add a mapping for our new pv_page and free its entrys (save one!) 615 * 616 * NOTE: If we are allocating a PV page for the kernel pmap, the 617 * pmap is already locked! (...but entering the mapping is safe...) 618 */ 619 620 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), VM_PROT_ALL); 621 pmap_update(pmap_kernel()); 622 pvpage = (struct pv_page *) pv_cachedva; 623 pv_cachedva = 0; 624 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 625 } 626 627 /* 628 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 629 * 630 * => caller must hold pvalloc_lock 631 * => if need_entry is true, we allocate and return one pv_entry 632 */ 633 634 static struct pv_entry * 635 pmap_add_pvpage(pvp, need_entry) 636 struct pv_page *pvp; 637 boolean_t need_entry; 638 { 639 int tofree, lcv; 640 641 /* do we need to return one? */ 642 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 643 644 pvp->pvinfo.pvpi_pvfree = NULL; 645 pvp->pvinfo.pvpi_nfree = tofree; 646 for (lcv = 0 ; lcv < tofree ; lcv++) { 647 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 648 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 649 } 650 if (need_entry) 651 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 652 else 653 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 654 pv_nfpvents += tofree; 655 return((need_entry) ? &pvp->pvents[lcv] : NULL); 656 } 657 658 /* 659 * pmap_free_pv_doit: actually free a pv_entry 660 * 661 * => do not call this directly! instead use either 662 * 1. pmap_free_pv ==> free a single pv_entry 663 * 2. pmap_free_pvs => free a list of pv_entrys 664 * => we must be holding pvalloc_lock 665 */ 666 667 __inline static void 668 pmap_free_pv_doit(pv) 669 struct pv_entry *pv; 670 { 671 struct pv_page *pvp; 672 673 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 674 pv_nfpvents++; 675 pvp->pvinfo.pvpi_nfree++; 676 677 /* nfree == 1 => fully allocated page just became partly allocated */ 678 if (pvp->pvinfo.pvpi_nfree == 1) { 679 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 680 } 681 682 /* free it */ 683 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 684 pvp->pvinfo.pvpi_pvfree = pv; 685 686 /* 687 * are all pv_page's pv_entry's free? move it to unused queue. 688 */ 689 690 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 691 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 692 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 693 } 694 } 695 696 /* 697 * pmap_free_pv: free a single pv_entry 698 * 699 * => we gain the pvalloc_lock 700 */ 701 702 __inline static void 703 pmap_free_pv(pmap, pv) 704 struct pmap *pmap; 705 struct pv_entry *pv; 706 { 707 simple_lock(&pvalloc_lock); 708 pmap_free_pv_doit(pv); 709 710 /* 711 * Can't free the PV page if the PV entries were associated with 712 * the kernel pmap; the pmap is already locked. 713 */ 714 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 715 pmap != pmap_kernel()) 716 pmap_free_pvpage(); 717 718 simple_unlock(&pvalloc_lock); 719 } 720 721 /* 722 * pmap_free_pvs: free a list of pv_entrys 723 * 724 * => we gain the pvalloc_lock 725 */ 726 727 __inline static void 728 pmap_free_pvs(pmap, pvs) 729 struct pmap *pmap; 730 struct pv_entry *pvs; 731 { 732 struct pv_entry *nextpv; 733 734 simple_lock(&pvalloc_lock); 735 736 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 737 nextpv = pvs->pv_next; 738 pmap_free_pv_doit(pvs); 739 } 740 741 /* 742 * Can't free the PV page if the PV entries were associated with 743 * the kernel pmap; the pmap is already locked. 744 */ 745 if (pv_nfpvents > PVE_HIWAT && pv_unusedpgs.tqh_first != NULL && 746 pmap != pmap_kernel()) 747 pmap_free_pvpage(); 748 749 simple_unlock(&pvalloc_lock); 750 } 751 752 753 /* 754 * pmap_free_pvpage: try and free an unused pv_page structure 755 * 756 * => assume caller is holding the pvalloc_lock and that 757 * there is a page on the pv_unusedpgs list 758 * => if we can't get a lock on the kmem_map we try again later 759 * => note: analysis of MI kmem_map usage [i.e. malloc/free] shows 760 * that if we can lock the kmem_map then we are not already 761 * holding kmem_object's lock. 762 */ 763 764 static void 765 pmap_free_pvpage() 766 { 767 int s; 768 struct vm_map *map; 769 struct vm_map_entry *dead_entries; 770 struct pv_page *pvp; 771 772 s = splvm(); /* protect kmem_map */ 773 774 pvp = pv_unusedpgs.tqh_first; 775 776 /* 777 * note: watch out for pv_initpage which is allocated out of 778 * kernel_map rather than kmem_map. 779 */ 780 if (pvp == pv_initpage) 781 map = kernel_map; 782 else 783 map = kmem_map; 784 785 if (vm_map_lock_try(map)) { 786 787 /* remove pvp from pv_unusedpgs */ 788 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 789 790 /* unmap the page */ 791 dead_entries = NULL; 792 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 793 &dead_entries); 794 vm_map_unlock(map); 795 796 if (dead_entries != NULL) 797 uvm_unmap_detach(dead_entries, 0); 798 799 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 800 } 801 802 if (pvp == pv_initpage) 803 /* no more initpage, we've freed it */ 804 pv_initpage = NULL; 805 806 splx(s); 807 } 808 809 /* 810 * main pv_entry manipulation functions: 811 * pmap_enter_pv: enter a mapping onto a vm_page list 812 * pmap_remove_pv: remove a mappiing from a vm_page list 813 * 814 * NOTE: pmap_enter_pv expects to lock the pvh itself 815 * pmap_remove_pv expects te caller to lock the pvh before calling 816 */ 817 818 /* 819 * pmap_enter_pv: enter a mapping onto a vm_page lst 820 * 821 * => caller should hold the proper lock on pmap_main_lock 822 * => caller should have pmap locked 823 * => we will gain the lock on the vm_page and allocate the new pv_entry 824 * => caller should adjust ptp's wire_count before calling 825 * => caller should not adjust pmap's wire_count 826 */ 827 828 __inline static void 829 pmap_enter_pv(pg, pve, pmap, va, ptp, flags) 830 struct vm_page *pg; 831 struct pv_entry *pve; /* preallocated pve for us to use */ 832 struct pmap *pmap; 833 vaddr_t va; 834 struct vm_page *ptp; /* PTP in pmap that maps this VA */ 835 int flags; 836 { 837 pve->pv_pmap = pmap; 838 pve->pv_va = va; 839 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 840 pve->pv_flags = flags; 841 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 842 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 843 pg->mdpage.pvh_list = pve; /* ... locked list */ 844 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 845 if (pve->pv_flags & PT_W) 846 ++pmap->pm_stats.wired_count; 847 } 848 849 /* 850 * pmap_remove_pv: try to remove a mapping from a pv_list 851 * 852 * => caller should hold proper lock on pmap_main_lock 853 * => pmap should be locked 854 * => caller should hold lock on vm_page [so that attrs can be adjusted] 855 * => caller should adjust ptp's wire_count and free PTP if needed 856 * => caller should NOT adjust pmap's wire_count 857 * => we return the removed pve 858 */ 859 860 __inline static struct pv_entry * 861 pmap_remove_pv(pg, pmap, va) 862 struct vm_page *pg; 863 struct pmap *pmap; 864 vaddr_t va; 865 { 866 struct pv_entry *pve, **prevptr; 867 868 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 869 pve = *prevptr; 870 while (pve) { 871 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 872 *prevptr = pve->pv_next; /* remove it! */ 873 if (pve->pv_flags & PT_W) 874 --pmap->pm_stats.wired_count; 875 break; 876 } 877 prevptr = &pve->pv_next; /* previous pointer */ 878 pve = pve->pv_next; /* advance */ 879 } 880 return(pve); /* return removed pve */ 881 } 882 883 /* 884 * 885 * pmap_modify_pv: Update pv flags 886 * 887 * => caller should hold lock on vm_page [so that attrs can be adjusted] 888 * => caller should NOT adjust pmap's wire_count 889 * => caller must call pmap_vac_me_harder() if writable status of a page 890 * may have changed. 891 * => we return the old flags 892 * 893 * Modify a physical-virtual mapping in the pv table 894 */ 895 896 /*__inline */ 897 static u_int 898 pmap_modify_pv(pmap, va, pg, bic_mask, eor_mask) 899 struct pmap *pmap; 900 vaddr_t va; 901 struct vm_page *pg; 902 u_int bic_mask; 903 u_int eor_mask; 904 { 905 struct pv_entry *npv; 906 u_int flags, oflags; 907 908 /* 909 * There is at least one VA mapping this page. 910 */ 911 912 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 913 if (pmap == npv->pv_pmap && va == npv->pv_va) { 914 oflags = npv->pv_flags; 915 npv->pv_flags = flags = 916 ((oflags & ~bic_mask) ^ eor_mask); 917 if ((flags ^ oflags) & PT_W) { 918 if (flags & PT_W) 919 ++pmap->pm_stats.wired_count; 920 else 921 --pmap->pm_stats.wired_count; 922 } 923 return (oflags); 924 } 925 } 926 return (0); 927 } 928 929 /* 930 * Map the specified level 2 pagetable into the level 1 page table for 931 * the given pmap to cover a chunk of virtual address space starting from the 932 * address specified. 933 */ 934 static /*__inline*/ void 935 pmap_map_in_l1(pmap, va, l2pa, selfref) 936 struct pmap *pmap; 937 vaddr_t va, l2pa; 938 boolean_t selfref; 939 { 940 vaddr_t ptva; 941 942 /* Calculate the index into the L1 page table. */ 943 ptva = (va >> PDSHIFT) & ~3; 944 945 NPDEBUG(PDB_MAP_L1, printf("wiring %08lx in to pd%p pte0x%lx va0x%lx\n", l2pa, 946 pmap->pm_pdir, L1_PTE(l2pa), ptva)); 947 948 /* Map page table into the L1. */ 949 pmap->pm_pdir[ptva + 0] = L1_PTE(l2pa + 0x000); 950 pmap->pm_pdir[ptva + 1] = L1_PTE(l2pa + 0x400); 951 pmap->pm_pdir[ptva + 2] = L1_PTE(l2pa + 0x800); 952 pmap->pm_pdir[ptva + 3] = L1_PTE(l2pa + 0xc00); 953 954 /* Map the page table into the page table area. */ 955 if (selfref) { 956 NPDEBUG(PDB_MAP_L1, printf("pt self reference %lx in %lx\n", 957 L2_PTE_NC_NB(l2pa, AP_KRW), pmap->pm_vptpt)); 958 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 959 L2_PTE_NC_NB(l2pa, AP_KRW); 960 } 961 /* XXX should be a purge */ 962 /* cpu_tlb_flushD();*/ 963 } 964 965 #if 0 966 static /*__inline*/ void 967 pmap_unmap_in_l1(pmap, va) 968 struct pmap *pmap; 969 vaddr_t va; 970 { 971 vaddr_t ptva; 972 973 /* Calculate the index into the L1 page table. */ 974 ptva = (va >> PDSHIFT) & ~3; 975 976 /* Unmap page table from the L1. */ 977 pmap->pm_pdir[ptva + 0] = 0; 978 pmap->pm_pdir[ptva + 1] = 0; 979 pmap->pm_pdir[ptva + 2] = 0; 980 pmap->pm_pdir[ptva + 3] = 0; 981 982 /* Unmap the page table from the page table area. */ 983 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 984 985 /* XXX should be a purge */ 986 /* cpu_tlb_flushD();*/ 987 } 988 #endif 989 990 /* 991 * Used to map a range of physical addresses into kernel 992 * virtual address space. 993 * 994 * For now, VM is already on, we only need to map the 995 * specified memory. 996 */ 997 vaddr_t 998 pmap_map(va, spa, epa, prot) 999 vaddr_t va, spa, epa; 1000 int prot; 1001 { 1002 while (spa < epa) { 1003 pmap_kenter_pa(va, spa, prot); 1004 va += NBPG; 1005 spa += NBPG; 1006 } 1007 pmap_update(pmap_kernel()); 1008 return(va); 1009 } 1010 1011 1012 /* 1013 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 1014 * 1015 * bootstrap the pmap system. This is called from initarm and allows 1016 * the pmap system to initailise any structures it requires. 1017 * 1018 * Currently this sets up the kernel_pmap that is statically allocated 1019 * and also allocated virtual addresses for certain page hooks. 1020 * Currently the only one page hook is allocated that is used 1021 * to zero physical pages of memory. 1022 * It also initialises the start and end address of the kernel data space. 1023 */ 1024 extern paddr_t physical_freestart; 1025 extern paddr_t physical_freeend; 1026 1027 char *boot_head; 1028 1029 void 1030 pmap_bootstrap(kernel_l1pt, kernel_ptpt) 1031 pd_entry_t *kernel_l1pt; 1032 pv_addr_t kernel_ptpt; 1033 { 1034 int loop; 1035 paddr_t start, end; 1036 #if NISADMA > 0 1037 paddr_t istart; 1038 psize_t isize; 1039 #endif 1040 1041 pmap_kernel()->pm_pdir = kernel_l1pt; 1042 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 1043 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 1044 simple_lock_init(&pmap_kernel()->pm_lock); 1045 pmap_kernel()->pm_obj.pgops = NULL; 1046 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 1047 pmap_kernel()->pm_obj.uo_npages = 0; 1048 pmap_kernel()->pm_obj.uo_refs = 1; 1049 1050 /* 1051 * Initialize PAGE_SIZE-dependent variables. 1052 */ 1053 uvm_setpagesize(); 1054 1055 npages = 0; 1056 loop = 0; 1057 while (loop < bootconfig.dramblocks) { 1058 start = (paddr_t)bootconfig.dram[loop].address; 1059 end = start + (bootconfig.dram[loop].pages * NBPG); 1060 if (start < physical_freestart) 1061 start = physical_freestart; 1062 if (end > physical_freeend) 1063 end = physical_freeend; 1064 #if 0 1065 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1066 #endif 1067 #if NISADMA > 0 1068 if (pmap_isa_dma_range_intersect(start, end - start, 1069 &istart, &isize)) { 1070 /* 1071 * Place the pages that intersect with the 1072 * ISA DMA range onto the ISA DMA free list. 1073 */ 1074 #if 0 1075 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1076 istart + isize - 1); 1077 #endif 1078 uvm_page_physload(atop(istart), 1079 atop(istart + isize), atop(istart), 1080 atop(istart + isize), VM_FREELIST_ISADMA); 1081 npages += atop(istart + isize) - atop(istart); 1082 1083 /* 1084 * Load the pieces that come before 1085 * the intersection into the default 1086 * free list. 1087 */ 1088 if (start < istart) { 1089 #if 0 1090 printf(" BEFORE 0x%lx -> 0x%lx\n", 1091 start, istart - 1); 1092 #endif 1093 uvm_page_physload(atop(start), 1094 atop(istart), atop(start), 1095 atop(istart), VM_FREELIST_DEFAULT); 1096 npages += atop(istart) - atop(start); 1097 } 1098 1099 /* 1100 * Load the pieces that come after 1101 * the intersection into the default 1102 * free list. 1103 */ 1104 if ((istart + isize) < end) { 1105 #if 0 1106 printf(" AFTER 0x%lx -> 0x%lx\n", 1107 (istart + isize), end - 1); 1108 #endif 1109 uvm_page_physload(atop(istart + isize), 1110 atop(end), atop(istart + isize), 1111 atop(end), VM_FREELIST_DEFAULT); 1112 npages += atop(end) - atop(istart + isize); 1113 } 1114 } else { 1115 uvm_page_physload(atop(start), atop(end), 1116 atop(start), atop(end), VM_FREELIST_DEFAULT); 1117 npages += atop(end) - atop(start); 1118 } 1119 #else /* NISADMA > 0 */ 1120 uvm_page_physload(atop(start), atop(end), 1121 atop(start), atop(end), VM_FREELIST_DEFAULT); 1122 npages += atop(end) - atop(start); 1123 #endif /* NISADMA > 0 */ 1124 ++loop; 1125 } 1126 1127 #ifdef MYCROFT_HACK 1128 printf("npages = %ld\n", npages); 1129 #endif 1130 1131 virtual_start = KERNEL_VM_BASE; 1132 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE - 1; 1133 1134 ALLOC_PAGE_HOOK(page_hook0, NBPG); 1135 ALLOC_PAGE_HOOK(page_hook1, NBPG); 1136 1137 /* 1138 * The mem special device needs a virtual hook but we don't 1139 * need a pte 1140 */ 1141 memhook = (char *)virtual_start; 1142 virtual_start += NBPG; 1143 1144 msgbufaddr = (caddr_t)virtual_start; 1145 msgbufpte = (pt_entry_t)pmap_pte(pmap_kernel(), virtual_start); 1146 virtual_start += round_page(MSGBUFSIZE); 1147 1148 /* 1149 * init the static-global locks and global lists. 1150 */ 1151 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1152 simple_lock_init(&pvalloc_lock); 1153 simple_lock_init(&pmaps_lock); 1154 LIST_INIT(&pmaps); 1155 TAILQ_INIT(&pv_freepages); 1156 TAILQ_INIT(&pv_unusedpgs); 1157 1158 /* 1159 * initialize the pmap pool. 1160 */ 1161 1162 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1163 0, pool_page_alloc_nointr, pool_page_free_nointr, M_VMPMAP); 1164 1165 cpu_dcache_wbinv_all(); 1166 } 1167 1168 /* 1169 * void pmap_init(void) 1170 * 1171 * Initialize the pmap module. 1172 * Called by vm_init() in vm/vm_init.c in order to initialise 1173 * any structures that the pmap system needs to map virtual memory. 1174 */ 1175 1176 extern int physmem; 1177 1178 void 1179 pmap_init() 1180 { 1181 1182 /* 1183 * Set the available memory vars - These do not map to real memory 1184 * addresses and cannot as the physical memory is fragmented. 1185 * They are used by ps for %mem calculations. 1186 * One could argue whether this should be the entire memory or just 1187 * the memory that is useable in a user process. 1188 */ 1189 avail_start = 0; 1190 avail_end = physmem * NBPG; 1191 1192 /* 1193 * now we need to free enough pv_entry structures to allow us to get 1194 * the kmem_map/kmem_object allocated and inited (done after this 1195 * function is finished). to do this we allocate one bootstrap page out 1196 * of kernel_map and use it to provide an initial pool of pv_entry 1197 * structures. we never free this page. 1198 */ 1199 1200 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1201 if (pv_initpage == NULL) 1202 panic("pmap_init: pv_initpage"); 1203 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1204 pv_nfpvents = 0; 1205 (void) pmap_add_pvpage(pv_initpage, FALSE); 1206 1207 pmap_initialized = TRUE; 1208 1209 /* Initialise our L1 page table queues and counters */ 1210 SIMPLEQ_INIT(&l1pt_static_queue); 1211 l1pt_static_queue_count = 0; 1212 l1pt_static_create_count = 0; 1213 SIMPLEQ_INIT(&l1pt_queue); 1214 l1pt_queue_count = 0; 1215 l1pt_create_count = 0; 1216 l1pt_reuse_count = 0; 1217 } 1218 1219 /* 1220 * pmap_postinit() 1221 * 1222 * This routine is called after the vm and kmem subsystems have been 1223 * initialised. This allows the pmap code to perform any initialisation 1224 * that can only be done one the memory allocation is in place. 1225 */ 1226 1227 void 1228 pmap_postinit() 1229 { 1230 int loop; 1231 struct l1pt *pt; 1232 1233 #ifdef PMAP_STATIC_L1S 1234 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1235 #else /* PMAP_STATIC_L1S */ 1236 for (loop = 0; loop < max_processes; ++loop) { 1237 #endif /* PMAP_STATIC_L1S */ 1238 /* Allocate a L1 page table */ 1239 pt = pmap_alloc_l1pt(); 1240 if (!pt) 1241 panic("Cannot allocate static L1 page tables\n"); 1242 1243 /* Clean it */ 1244 bzero((void *)pt->pt_va, PD_SIZE); 1245 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1246 /* Add the page table to the queue */ 1247 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1248 ++l1pt_static_queue_count; 1249 ++l1pt_static_create_count; 1250 } 1251 } 1252 1253 1254 /* 1255 * Create and return a physical map. 1256 * 1257 * If the size specified for the map is zero, the map is an actual physical 1258 * map, and may be referenced by the hardware. 1259 * 1260 * If the size specified is non-zero, the map will be used in software only, 1261 * and is bounded by that size. 1262 */ 1263 1264 pmap_t 1265 pmap_create() 1266 { 1267 struct pmap *pmap; 1268 1269 /* 1270 * Fetch pmap entry from the pool 1271 */ 1272 1273 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1274 /* XXX is this really needed! */ 1275 memset(pmap, 0, sizeof(*pmap)); 1276 1277 simple_lock_init(&pmap->pm_obj.vmobjlock); 1278 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1279 TAILQ_INIT(&pmap->pm_obj.memq); 1280 pmap->pm_obj.uo_npages = 0; 1281 pmap->pm_obj.uo_refs = 1; 1282 pmap->pm_stats.wired_count = 0; 1283 pmap->pm_stats.resident_count = 1; 1284 1285 /* Now init the machine part of the pmap */ 1286 pmap_pinit(pmap); 1287 return(pmap); 1288 } 1289 1290 /* 1291 * pmap_alloc_l1pt() 1292 * 1293 * This routine allocates physical and virtual memory for a L1 page table 1294 * and wires it. 1295 * A l1pt structure is returned to describe the allocated page table. 1296 * 1297 * This routine is allowed to fail if the required memory cannot be allocated. 1298 * In this case NULL is returned. 1299 */ 1300 1301 struct l1pt * 1302 pmap_alloc_l1pt(void) 1303 { 1304 paddr_t pa; 1305 vaddr_t va; 1306 struct l1pt *pt; 1307 int error; 1308 struct vm_page *m; 1309 pt_entry_t *ptes; 1310 1311 /* Allocate virtual address space for the L1 page table */ 1312 va = uvm_km_valloc(kernel_map, PD_SIZE); 1313 if (va == 0) { 1314 #ifdef DIAGNOSTIC 1315 PDEBUG(0, 1316 printf("pmap: Cannot allocate pageable memory for L1\n")); 1317 #endif /* DIAGNOSTIC */ 1318 return(NULL); 1319 } 1320 1321 /* Allocate memory for the l1pt structure */ 1322 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1323 1324 /* 1325 * Allocate pages from the VM system. 1326 */ 1327 TAILQ_INIT(&pt->pt_plist); 1328 error = uvm_pglistalloc(PD_SIZE, physical_start, physical_end, 1329 PD_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1330 if (error) { 1331 #ifdef DIAGNOSTIC 1332 PDEBUG(0, 1333 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1334 error)); 1335 #endif /* DIAGNOSTIC */ 1336 /* Release the resources we already have claimed */ 1337 free(pt, M_VMPMAP); 1338 uvm_km_free(kernel_map, va, PD_SIZE); 1339 return(NULL); 1340 } 1341 1342 /* Map our physical pages into our virtual space */ 1343 pt->pt_va = va; 1344 m = pt->pt_plist.tqh_first; 1345 ptes = pmap_map_ptes(pmap_kernel()); 1346 while (m && va < (pt->pt_va + PD_SIZE)) { 1347 pa = VM_PAGE_TO_PHYS(m); 1348 1349 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1350 1351 /* Revoke cacheability and bufferability */ 1352 /* XXX should be done better than this */ 1353 ptes[arm_byte_to_page(va)] &= ~(PT_C | PT_B); 1354 1355 va += NBPG; 1356 m = m->pageq.tqe_next; 1357 } 1358 pmap_unmap_ptes(pmap_kernel()); 1359 pmap_update(pmap_kernel()); 1360 1361 #ifdef DIAGNOSTIC 1362 if (m) 1363 panic("pmap_alloc_l1pt: pglist not empty\n"); 1364 #endif /* DIAGNOSTIC */ 1365 1366 pt->pt_flags = 0; 1367 return(pt); 1368 } 1369 1370 /* 1371 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1372 */ 1373 static void 1374 pmap_free_l1pt(pt) 1375 struct l1pt *pt; 1376 { 1377 /* Separate the physical memory for the virtual space */ 1378 pmap_kremove(pt->pt_va, PD_SIZE); 1379 pmap_update(pmap_kernel()); 1380 1381 /* Return the physical memory */ 1382 uvm_pglistfree(&pt->pt_plist); 1383 1384 /* Free the virtual space */ 1385 uvm_km_free(kernel_map, pt->pt_va, PD_SIZE); 1386 1387 /* Free the l1pt structure */ 1388 free(pt, M_VMPMAP); 1389 } 1390 1391 /* 1392 * Allocate a page directory. 1393 * This routine will either allocate a new page directory from the pool 1394 * of L1 page tables currently held by the kernel or it will allocate 1395 * a new one via pmap_alloc_l1pt(). 1396 * It will then initialise the l1 page table for use. 1397 * 1398 * XXX must tidy up and fix this code, not happy about how it does the pmaps_locking 1399 */ 1400 static int 1401 pmap_allocpagedir(pmap) 1402 struct pmap *pmap; 1403 { 1404 paddr_t pa; 1405 struct l1pt *pt; 1406 pt_entry_t *pte; 1407 1408 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1409 1410 /* Do we have any spare L1's lying around ? */ 1411 if (l1pt_static_queue_count) { 1412 --l1pt_static_queue_count; 1413 pt = l1pt_static_queue.sqh_first; 1414 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1415 } else if (l1pt_queue_count) { 1416 --l1pt_queue_count; 1417 pt = l1pt_queue.sqh_first; 1418 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1419 ++l1pt_reuse_count; 1420 } else { 1421 pt = pmap_alloc_l1pt(); 1422 if (!pt) 1423 return(ENOMEM); 1424 ++l1pt_create_count; 1425 } 1426 1427 /* Store the pointer to the l1 descriptor in the pmap. */ 1428 pmap->pm_l1pt = pt; 1429 1430 /* Get the physical address of the start of the l1 */ 1431 pa = VM_PAGE_TO_PHYS(pt->pt_plist.tqh_first); 1432 1433 /* Store the virtual address of the l1 in the pmap. */ 1434 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1435 1436 /* Clean the L1 if it is dirty */ 1437 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1438 bzero((void *)pmap->pm_pdir, (PD_SIZE - KERNEL_PD_SIZE)); 1439 1440 /* Allocate a page table to map all the page tables for this pmap */ 1441 1442 #ifdef DIAGNOSTIC 1443 if (pmap->pm_vptpt) { 1444 /* XXX What if we have one already ? */ 1445 panic("pmap_allocpagedir: have pt already\n"); 1446 } 1447 #endif /* DIAGNOSTIC */ 1448 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1449 if (pmap->pm_vptpt == 0) { 1450 pmap_freepagedir(pmap); 1451 return(ENOMEM); 1452 } 1453 1454 /* need to lock this all up for growkernel */ 1455 simple_lock(&pmaps_lock); 1456 /* wish we didn't have to keep this locked... */ 1457 1458 /* Duplicate the kernel mapping i.e. all mappings 0xf0000000+ */ 1459 bcopy((char *)pmap_kernel()->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1460 (char *)pmap->pm_pdir + (PD_SIZE - KERNEL_PD_SIZE), 1461 KERNEL_PD_SIZE); 1462 1463 (void) pmap_extract(pmap_kernel(), pmap->pm_vptpt, &pmap->pm_pptpt); 1464 pmap->pm_pptpt &= PG_FRAME; 1465 /* Revoke cacheability and bufferability */ 1466 /* XXX should be done better than this */ 1467 pte = pmap_pte(pmap_kernel(), pmap->pm_vptpt); 1468 *pte = *pte & ~(PT_C | PT_B); 1469 1470 /* Wire in this page table */ 1471 pmap_map_in_l1(pmap, PROCESS_PAGE_TBLS_BASE, pmap->pm_pptpt, TRUE); 1472 1473 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1474 1475 /* 1476 * Map the kernel page tables for 0xf0000000 + 1477 * into the page table used to map the 1478 * pmap's page tables 1479 */ 1480 bcopy((char *)(PROCESS_PAGE_TBLS_BASE 1481 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) 1482 + ((PD_SIZE - KERNEL_PD_SIZE) >> 2)), 1483 (char *)pmap->pm_vptpt + ((PD_SIZE - KERNEL_PD_SIZE) >> 2), 1484 (KERNEL_PD_SIZE >> 2)); 1485 1486 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1487 simple_unlock(&pmaps_lock); 1488 1489 return(0); 1490 } 1491 1492 1493 /* 1494 * Initialize a preallocated and zeroed pmap structure, 1495 * such as one in a vmspace structure. 1496 */ 1497 1498 void 1499 pmap_pinit(pmap) 1500 struct pmap *pmap; 1501 { 1502 int backoff = 6; 1503 int retry = 10; 1504 1505 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1506 1507 /* Keep looping until we succeed in allocating a page directory */ 1508 while (pmap_allocpagedir(pmap) != 0) { 1509 /* 1510 * Ok we failed to allocate a suitable block of memory for an 1511 * L1 page table. This means that either: 1512 * 1. 16KB of virtual address space could not be allocated 1513 * 2. 16KB of physically contiguous memory on a 16KB boundary 1514 * could not be allocated. 1515 * 1516 * Since we cannot fail we will sleep for a while and try 1517 * again. 1518 * 1519 * Searching for a suitable L1 PT is expensive: 1520 * to avoid hogging the system when memory is really 1521 * scarce, use an exponential back-off so that 1522 * eventually we won't retry more than once every 8 1523 * seconds. This should allow other processes to run 1524 * to completion and free up resources. 1525 */ 1526 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1527 NULL); 1528 if (--retry == 0) { 1529 retry = 10; 1530 if (backoff) 1531 --backoff; 1532 } 1533 } 1534 1535 /* Map zero page for the pmap. This will also map the L2 for it */ 1536 pmap_enter(pmap, 0x00000000, systempage.pv_pa, 1537 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1538 pmap_update(pmap); 1539 } 1540 1541 1542 void 1543 pmap_freepagedir(pmap) 1544 struct pmap *pmap; 1545 { 1546 /* Free the memory used for the page table mapping */ 1547 if (pmap->pm_vptpt != 0) 1548 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1549 1550 /* junk the L1 page table */ 1551 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1552 /* Add the page table to the queue */ 1553 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1554 ++l1pt_static_queue_count; 1555 } else if (l1pt_queue_count < 8) { 1556 /* Add the page table to the queue */ 1557 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1558 ++l1pt_queue_count; 1559 } else 1560 pmap_free_l1pt(pmap->pm_l1pt); 1561 } 1562 1563 1564 /* 1565 * Retire the given physical map from service. 1566 * Should only be called if the map contains no valid mappings. 1567 */ 1568 1569 void 1570 pmap_destroy(pmap) 1571 struct pmap *pmap; 1572 { 1573 struct vm_page *page; 1574 int count; 1575 1576 if (pmap == NULL) 1577 return; 1578 1579 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1580 1581 /* 1582 * Drop reference count 1583 */ 1584 simple_lock(&pmap->pm_obj.vmobjlock); 1585 count = --pmap->pm_obj.uo_refs; 1586 simple_unlock(&pmap->pm_obj.vmobjlock); 1587 if (count > 0) { 1588 return; 1589 } 1590 1591 /* 1592 * reference count is zero, free pmap resources and then free pmap. 1593 */ 1594 1595 /* 1596 * remove it from global list of pmaps 1597 */ 1598 1599 simple_lock(&pmaps_lock); 1600 LIST_REMOVE(pmap, pm_list); 1601 simple_unlock(&pmaps_lock); 1602 1603 /* Remove the zero page mapping */ 1604 pmap_remove(pmap, 0x00000000, 0x00000000 + NBPG); 1605 pmap_update(pmap); 1606 1607 /* 1608 * Free any page tables still mapped 1609 * This is only temporay until pmap_enter can count the number 1610 * of mappings made in a page table. Then pmap_remove() can 1611 * reduce the count and free the pagetable when the count 1612 * reaches zero. Note that entries in this list should match the 1613 * contents of the ptpt, however this is faster than walking a 1024 1614 * entries looking for pt's 1615 * taken from i386 pmap.c 1616 */ 1617 while (pmap->pm_obj.memq.tqh_first != NULL) { 1618 page = pmap->pm_obj.memq.tqh_first; 1619 #ifdef DIAGNOSTIC 1620 if (page->flags & PG_BUSY) 1621 panic("pmap_release: busy page table page"); 1622 #endif 1623 /* pmap_page_protect? currently no need for it. */ 1624 1625 page->wire_count = 0; 1626 uvm_pagefree(page); 1627 } 1628 1629 /* Free the page dir */ 1630 pmap_freepagedir(pmap); 1631 1632 /* return the pmap to the pool */ 1633 pool_put(&pmap_pmap_pool, pmap); 1634 } 1635 1636 1637 /* 1638 * void pmap_reference(struct pmap *pmap) 1639 * 1640 * Add a reference to the specified pmap. 1641 */ 1642 1643 void 1644 pmap_reference(pmap) 1645 struct pmap *pmap; 1646 { 1647 if (pmap == NULL) 1648 return; 1649 1650 simple_lock(&pmap->pm_lock); 1651 pmap->pm_obj.uo_refs++; 1652 simple_unlock(&pmap->pm_lock); 1653 } 1654 1655 /* 1656 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1657 * 1658 * Return the start and end addresses of the kernel's virtual space. 1659 * These values are setup in pmap_bootstrap and are updated as pages 1660 * are allocated. 1661 */ 1662 1663 void 1664 pmap_virtual_space(start, end) 1665 vaddr_t *start; 1666 vaddr_t *end; 1667 { 1668 *start = virtual_start; 1669 *end = virtual_end; 1670 } 1671 1672 1673 /* 1674 * Activate the address space for the specified process. If the process 1675 * is the current process, load the new MMU context. 1676 */ 1677 void 1678 pmap_activate(p) 1679 struct proc *p; 1680 { 1681 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1682 struct pcb *pcb = &p->p_addr->u_pcb; 1683 1684 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1685 (paddr_t *)&pcb->pcb_pagedir); 1686 1687 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1688 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1689 1690 if (p == curproc) { 1691 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1692 setttb((u_int)pcb->pcb_pagedir); 1693 } 1694 #if 0 1695 pmap->pm_pdchanged = FALSE; 1696 #endif 1697 } 1698 1699 1700 /* 1701 * Deactivate the address space of the specified process. 1702 */ 1703 void 1704 pmap_deactivate(p) 1705 struct proc *p; 1706 { 1707 } 1708 1709 /* 1710 * Perform any deferred pmap operations. 1711 */ 1712 void 1713 pmap_update(struct pmap *pmap) 1714 { 1715 1716 /* 1717 * We haven't deferred any pmap operations, but we do need to 1718 * make sure TLB/cache operations have completed. 1719 */ 1720 cpu_cpwait(); 1721 } 1722 1723 /* 1724 * pmap_clean_page() 1725 * 1726 * This is a local function used to work out the best strategy to clean 1727 * a single page referenced by its entry in the PV table. It's used by 1728 * pmap_copy_page, pmap_zero page and maybe some others later on. 1729 * 1730 * Its policy is effectively: 1731 * o If there are no mappings, we don't bother doing anything with the cache. 1732 * o If there is one mapping, we clean just that page. 1733 * o If there are multiple mappings, we clean the entire cache. 1734 * 1735 * So that some functions can be further optimised, it returns 0 if it didn't 1736 * clean the entire cache, or 1 if it did. 1737 * 1738 * XXX One bug in this routine is that if the pv_entry has a single page 1739 * mapped at 0x00000000 a whole cache clean will be performed rather than 1740 * just the 1 page. Since this should not occur in everyday use and if it does 1741 * it will just result in not the most efficient clean for the page. 1742 */ 1743 static int 1744 pmap_clean_page(pv, is_src) 1745 struct pv_entry *pv; 1746 boolean_t is_src; 1747 { 1748 struct pmap *pmap; 1749 struct pv_entry *npv; 1750 int cache_needs_cleaning = 0; 1751 vaddr_t page_to_clean = 0; 1752 1753 if (pv == NULL) 1754 /* nothing mapped in so nothing to flush */ 1755 return (0); 1756 1757 /* Since we flush the cache each time we change curproc, we 1758 * only need to flush the page if it is in the current pmap. 1759 */ 1760 if (curproc) 1761 pmap = curproc->p_vmspace->vm_map.pmap; 1762 else 1763 pmap = pmap_kernel(); 1764 1765 for (npv = pv; npv; npv = npv->pv_next) { 1766 if (npv->pv_pmap == pmap) { 1767 /* The page is mapped non-cacheable in 1768 * this map. No need to flush the cache. 1769 */ 1770 if (npv->pv_flags & PT_NC) { 1771 #ifdef DIAGNOSTIC 1772 if (cache_needs_cleaning) 1773 panic("pmap_clean_page: " 1774 "cache inconsistency"); 1775 #endif 1776 break; 1777 } 1778 #if 0 1779 /* This doesn't work, because pmap_protect 1780 doesn't flush changes on pages that it 1781 has write-protected. */ 1782 1783 /* If the page is not writable and this 1784 is the source, then there is no need 1785 to flush it from the cache. */ 1786 else if (is_src && ! (npv->pv_flags & PT_Wr)) 1787 continue; 1788 #endif 1789 if (cache_needs_cleaning){ 1790 page_to_clean = 0; 1791 break; 1792 } 1793 else 1794 page_to_clean = npv->pv_va; 1795 cache_needs_cleaning = 1; 1796 } 1797 } 1798 1799 if (page_to_clean) 1800 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1801 else if (cache_needs_cleaning) { 1802 cpu_idcache_wbinv_all(); 1803 return (1); 1804 } 1805 return (0); 1806 } 1807 1808 /* 1809 * pmap_zero_page() 1810 * 1811 * Zero a given physical page by mapping it at a page hook point. 1812 * In doing the zero page op, the page we zero is mapped cachable, as with 1813 * StrongARM accesses to non-cached pages are non-burst making writing 1814 * _any_ bulk data very slow. 1815 */ 1816 void 1817 pmap_zero_page(phys) 1818 paddr_t phys; 1819 { 1820 struct vm_page *pg; 1821 1822 /* Get an entry for this page, and clean it it. */ 1823 pg = PHYS_TO_VM_PAGE(phys); 1824 simple_lock(&pg->mdpage.pvh_slock); 1825 pmap_clean_page(pg->mdpage.pvh_list, FALSE); 1826 simple_unlock(&pg->mdpage.pvh_slock); 1827 1828 /* 1829 * Hook in the page, zero it, and purge the cache for that 1830 * zeroed page. Invalidate the TLB as needed. 1831 */ 1832 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1833 cpu_tlb_flushD_SE(page_hook0.va); 1834 cpu_cpwait(); 1835 bzero_page(page_hook0.va); 1836 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1837 } 1838 1839 /* pmap_pageidlezero() 1840 * 1841 * The same as above, except that we assume that the page is not 1842 * mapped. This means we never have to flush the cache first. Called 1843 * from the idle loop. 1844 */ 1845 boolean_t 1846 pmap_pageidlezero(phys) 1847 paddr_t phys; 1848 { 1849 int i, *ptr; 1850 boolean_t rv = TRUE; 1851 1852 #ifdef DIAGNOSTIC 1853 struct vm_page *pg; 1854 1855 pg = PHYS_TO_VM_PAGE(phys); 1856 if (pg->mdpage.pvh_list != NULL) 1857 panic("pmap_pageidlezero: zeroing mapped page\n"); 1858 #endif 1859 1860 /* 1861 * Hook in the page, zero it, and purge the cache for that 1862 * zeroed page. Invalidate the TLB as needed. 1863 */ 1864 *page_hook0.pte = L2_PTE(phys & PG_FRAME, AP_KRW); 1865 cpu_tlb_flushD_SE(page_hook0.va); 1866 cpu_cpwait(); 1867 1868 for (i = 0, ptr = (int *)page_hook0.va; 1869 i < (NBPG / sizeof(int)); i++) { 1870 if (sched_whichqs != 0) { 1871 /* 1872 * A process has become ready. Abort now, 1873 * so we don't keep it waiting while we 1874 * do slow memory access to finish this 1875 * page. 1876 */ 1877 rv = FALSE; 1878 break; 1879 } 1880 *ptr++ = 0; 1881 } 1882 1883 if (rv) 1884 /* 1885 * if we aborted we'll rezero this page again later so don't 1886 * purge it unless we finished it 1887 */ 1888 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1889 return (rv); 1890 } 1891 1892 /* 1893 * pmap_copy_page() 1894 * 1895 * Copy one physical page into another, by mapping the pages into 1896 * hook points. The same comment regarding cachability as in 1897 * pmap_zero_page also applies here. 1898 */ 1899 void 1900 pmap_copy_page(src, dest) 1901 paddr_t src; 1902 paddr_t dest; 1903 { 1904 struct vm_page *src_pg, *dest_pg; 1905 boolean_t cleanedcache; 1906 1907 /* Get PV entries for the pages, and clean them if needed. */ 1908 src_pg = PHYS_TO_VM_PAGE(src); 1909 1910 simple_lock(&src_pg->mdpage.pvh_slock); 1911 cleanedcache = pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1912 simple_unlock(&src_pg->mdpage.pvh_slock); 1913 1914 if (cleanedcache == 0) { 1915 dest_pg = PHYS_TO_VM_PAGE(dest); 1916 simple_lock(&dest_pg->mdpage.pvh_slock); 1917 pmap_clean_page(dest_pg->mdpage.pvh_list, FALSE); 1918 simple_unlock(&dest_pg->mdpage.pvh_slock); 1919 } 1920 /* 1921 * Map the pages into the page hook points, copy them, and purge 1922 * the cache for the appropriate page. Invalidate the TLB 1923 * as required. 1924 */ 1925 *page_hook0.pte = L2_PTE(src & PG_FRAME, AP_KRW); 1926 *page_hook1.pte = L2_PTE(dest & PG_FRAME, AP_KRW); 1927 cpu_tlb_flushD_SE(page_hook0.va); 1928 cpu_tlb_flushD_SE(page_hook1.va); 1929 cpu_cpwait(); 1930 bcopy_page(page_hook0.va, page_hook1.va); 1931 cpu_dcache_wbinv_range(page_hook0.va, NBPG); 1932 cpu_dcache_wbinv_range(page_hook1.va, NBPG); 1933 } 1934 1935 #if 0 1936 void 1937 pmap_pte_addref(pmap, va) 1938 struct pmap *pmap; 1939 vaddr_t va; 1940 { 1941 pd_entry_t *pde; 1942 paddr_t pa; 1943 struct vm_page *m; 1944 1945 if (pmap == pmap_kernel()) 1946 return; 1947 1948 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1949 pa = pmap_pte_pa(pde); 1950 m = PHYS_TO_VM_PAGE(pa); 1951 ++m->wire_count; 1952 #ifdef MYCROFT_HACK 1953 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1954 pmap, va, pde, pa, m, m->wire_count); 1955 #endif 1956 } 1957 1958 void 1959 pmap_pte_delref(pmap, va) 1960 struct pmap *pmap; 1961 vaddr_t va; 1962 { 1963 pd_entry_t *pde; 1964 paddr_t pa; 1965 struct vm_page *m; 1966 1967 if (pmap == pmap_kernel()) 1968 return; 1969 1970 pde = pmap_pde(pmap, va & ~(3 << PDSHIFT)); 1971 pa = pmap_pte_pa(pde); 1972 m = PHYS_TO_VM_PAGE(pa); 1973 --m->wire_count; 1974 #ifdef MYCROFT_HACK 1975 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1976 pmap, va, pde, pa, m, m->wire_count); 1977 #endif 1978 if (m->wire_count == 0) { 1979 #ifdef MYCROFT_HACK 1980 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1981 pmap, va, pde, pa, m); 1982 #endif 1983 pmap_unmap_in_l1(pmap, va); 1984 uvm_pagefree(m); 1985 --pmap->pm_stats.resident_count; 1986 } 1987 } 1988 #else 1989 #define pmap_pte_addref(pmap, va) 1990 #define pmap_pte_delref(pmap, va) 1991 #endif 1992 1993 /* 1994 * Since we have a virtually indexed cache, we may need to inhibit caching if 1995 * there is more than one mapping and at least one of them is writable. 1996 * Since we purge the cache on every context switch, we only need to check for 1997 * other mappings within the same pmap, or kernel_pmap. 1998 * This function is also called when a page is unmapped, to possibly reenable 1999 * caching on any remaining mappings. 2000 * 2001 * The code implements the following logic, where: 2002 * 2003 * KW = # of kernel read/write pages 2004 * KR = # of kernel read only pages 2005 * UW = # of user read/write pages 2006 * UR = # of user read only pages 2007 * OW = # of user read/write pages in another pmap, then 2008 * 2009 * KC = kernel mapping is cacheable 2010 * UC = user mapping is cacheable 2011 * 2012 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2013 * +--------------------------------------------- 2014 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2015 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2016 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2017 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2018 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2019 * 2020 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2021 */ 2022 __inline static void 2023 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2024 boolean_t clear_cache) 2025 { 2026 if (pmap == pmap_kernel()) 2027 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2028 else 2029 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2030 } 2031 2032 static void 2033 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2034 boolean_t clear_cache) 2035 { 2036 int user_entries = 0; 2037 int user_writable = 0; 2038 int user_cacheable = 0; 2039 int kernel_entries = 0; 2040 int kernel_writable = 0; 2041 int kernel_cacheable = 0; 2042 struct pv_entry *pv; 2043 struct pmap *last_pmap = pmap; 2044 2045 #ifdef DIAGNOSTIC 2046 if (pmap != pmap_kernel()) 2047 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2048 #endif 2049 2050 /* 2051 * Pass one, see if there are both kernel and user pmaps for 2052 * this page. Calculate whether there are user-writable or 2053 * kernel-writable pages. 2054 */ 2055 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2056 if (pv->pv_pmap != pmap) { 2057 user_entries++; 2058 if (pv->pv_flags & PT_Wr) 2059 user_writable++; 2060 if ((pv->pv_flags & PT_NC) == 0) 2061 user_cacheable++; 2062 } else { 2063 kernel_entries++; 2064 if (pv->pv_flags & PT_Wr) 2065 kernel_writable++; 2066 if ((pv->pv_flags & PT_NC) == 0) 2067 kernel_cacheable++; 2068 } 2069 } 2070 2071 /* 2072 * We know we have just been updating a kernel entry, so if 2073 * all user pages are already cacheable, then there is nothing 2074 * further to do. 2075 */ 2076 if (kernel_entries == 0 && 2077 user_cacheable == user_entries) 2078 return; 2079 2080 if (user_entries) { 2081 /* 2082 * Scan over the list again, for each entry, if it 2083 * might not be set correctly, call pmap_vac_me_user 2084 * to recalculate the settings. 2085 */ 2086 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2087 /* 2088 * We know kernel mappings will get set 2089 * correctly in other calls. We also know 2090 * that if the pmap is the same as last_pmap 2091 * then we've just handled this entry. 2092 */ 2093 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2094 continue; 2095 /* 2096 * If there are kernel entries and this page 2097 * is writable but non-cacheable, then we can 2098 * skip this entry also. 2099 */ 2100 if (kernel_entries > 0 && 2101 (pv->pv_flags & (PT_NC | PT_Wr)) == 2102 (PT_NC | PT_Wr)) 2103 continue; 2104 /* 2105 * Similarly if there are no kernel-writable 2106 * entries and the page is already 2107 * read-only/cacheable. 2108 */ 2109 if (kernel_writable == 0 && 2110 (pv->pv_flags & (PT_NC | PT_Wr)) == 0) 2111 continue; 2112 /* 2113 * For some of the remaining cases, we know 2114 * that we must recalculate, but for others we 2115 * can't tell if they are correct or not, so 2116 * we recalculate anyway. 2117 */ 2118 pmap_unmap_ptes(last_pmap); 2119 last_pmap = pv->pv_pmap; 2120 ptes = pmap_map_ptes(last_pmap); 2121 pmap_vac_me_user(last_pmap, pg, ptes, 2122 pmap_is_curpmap(last_pmap)); 2123 } 2124 /* Restore the pte mapping that was passed to us. */ 2125 if (last_pmap != pmap) { 2126 pmap_unmap_ptes(last_pmap); 2127 ptes = pmap_map_ptes(pmap); 2128 } 2129 if (kernel_entries == 0) 2130 return; 2131 } 2132 2133 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2134 return; 2135 } 2136 2137 static void 2138 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2139 boolean_t clear_cache) 2140 { 2141 struct pmap *kpmap = pmap_kernel(); 2142 struct pv_entry *pv, *npv; 2143 int entries = 0; 2144 int writable = 0; 2145 int cacheable_entries = 0; 2146 int kern_cacheable = 0; 2147 int other_writable = 0; 2148 2149 pv = pg->mdpage.pvh_list; 2150 KASSERT(ptes != NULL); 2151 2152 /* 2153 * Count mappings and writable mappings in this pmap. 2154 * Include kernel mappings as part of our own. 2155 * Keep a pointer to the first one. 2156 */ 2157 for (npv = pv; npv; npv = npv->pv_next) { 2158 /* Count mappings in the same pmap */ 2159 if (pmap == npv->pv_pmap || 2160 kpmap == npv->pv_pmap) { 2161 if (entries++ == 0) 2162 pv = npv; 2163 /* Cacheable mappings */ 2164 if ((npv->pv_flags & PT_NC) == 0) { 2165 cacheable_entries++; 2166 if (kpmap == npv->pv_pmap) 2167 kern_cacheable++; 2168 } 2169 /* Writable mappings */ 2170 if (npv->pv_flags & PT_Wr) 2171 ++writable; 2172 } else if (npv->pv_flags & PT_Wr) 2173 other_writable = 1; 2174 } 2175 2176 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2177 "writable %d cacheable %d %s\n", pmap, entries, writable, 2178 cacheable_entries, clear_cache ? "clean" : "no clean")); 2179 2180 /* 2181 * Enable or disable caching as necessary. 2182 * Note: the first entry might be part of the kernel pmap, 2183 * so we can't assume this is indicative of the state of the 2184 * other (maybe non-kpmap) entries. 2185 */ 2186 if ((entries > 1 && writable) || 2187 (entries > 0 && pmap == kpmap && other_writable)) { 2188 if (cacheable_entries == 0) 2189 return; 2190 for (npv = pv; npv; npv = npv->pv_next) { 2191 if ((pmap == npv->pv_pmap 2192 || kpmap == npv->pv_pmap) && 2193 (npv->pv_flags & PT_NC) == 0) { 2194 ptes[arm_byte_to_page(npv->pv_va)] &= 2195 ~(PT_C | PT_B); 2196 npv->pv_flags |= PT_NC; 2197 /* 2198 * If this page needs flushing from the 2199 * cache, and we aren't going to do it 2200 * below, do it now. 2201 */ 2202 if ((cacheable_entries < 4 && 2203 (clear_cache || npv->pv_pmap == kpmap)) || 2204 (npv->pv_pmap == kpmap && 2205 !clear_cache && kern_cacheable < 4)) { 2206 cpu_idcache_wbinv_range(npv->pv_va, 2207 NBPG); 2208 cpu_tlb_flushID_SE(npv->pv_va); 2209 } 2210 } 2211 } 2212 if ((clear_cache && cacheable_entries >= 4) || 2213 kern_cacheable >= 4) { 2214 cpu_idcache_wbinv_all(); 2215 cpu_tlb_flushID(); 2216 } 2217 cpu_cpwait(); 2218 } else if (entries > 0) { 2219 /* 2220 * Turn cacheing back on for some pages. If it is a kernel 2221 * page, only do so if there are no other writable pages. 2222 */ 2223 for (npv = pv; npv; npv = npv->pv_next) { 2224 if ((pmap == npv->pv_pmap || 2225 (kpmap == npv->pv_pmap && other_writable == 0)) && 2226 (npv->pv_flags & PT_NC)) { 2227 ptes[arm_byte_to_page(npv->pv_va)] |= 2228 pte_cache_mode; 2229 npv->pv_flags &= ~PT_NC; 2230 } 2231 } 2232 } 2233 } 2234 2235 /* 2236 * pmap_remove() 2237 * 2238 * pmap_remove is responsible for nuking a number of mappings for a range 2239 * of virtual address space in the current pmap. To do this efficiently 2240 * is interesting, because in a number of cases a wide virtual address 2241 * range may be supplied that contains few actual mappings. So, the 2242 * optimisations are: 2243 * 1. Try and skip over hunks of address space for which an L1 entry 2244 * does not exist. 2245 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2246 * maybe do just a partial cache clean. This path of execution is 2247 * complicated by the fact that the cache must be flushed _before_ 2248 * the PTE is nuked, being a VAC :-) 2249 * 3. Maybe later fast-case a single page, but I don't think this is 2250 * going to make _that_ much difference overall. 2251 */ 2252 2253 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2254 2255 void 2256 pmap_remove(pmap, sva, eva) 2257 struct pmap *pmap; 2258 vaddr_t sva; 2259 vaddr_t eva; 2260 { 2261 int cleanlist_idx = 0; 2262 struct pagelist { 2263 vaddr_t va; 2264 pt_entry_t *pte; 2265 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2266 pt_entry_t *pte = 0, *ptes; 2267 paddr_t pa; 2268 int pmap_active; 2269 struct vm_page *pg; 2270 2271 /* Exit quick if there is no pmap */ 2272 if (!pmap) 2273 return; 2274 2275 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", pmap, sva, eva)); 2276 2277 sva &= PG_FRAME; 2278 eva &= PG_FRAME; 2279 2280 /* 2281 * we lock in the pmap => vm_page direction 2282 */ 2283 PMAP_MAP_TO_HEAD_LOCK(); 2284 2285 ptes = pmap_map_ptes(pmap); 2286 /* Get a page table pointer */ 2287 while (sva < eva) { 2288 if (pmap_pde_page(pmap_pde(pmap, sva))) 2289 break; 2290 sva = (sva & PD_MASK) + NBPD; 2291 } 2292 2293 pte = &ptes[arm_byte_to_page(sva)]; 2294 /* Note if the pmap is active thus require cache and tlb cleans */ 2295 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2296 || (pmap == pmap_kernel())) 2297 pmap_active = 1; 2298 else 2299 pmap_active = 0; 2300 2301 /* Now loop along */ 2302 while (sva < eva) { 2303 /* Check if we can move to the next PDE (l1 chunk) */ 2304 if (!(sva & PT_MASK)) 2305 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2306 sva += NBPD; 2307 pte += arm_byte_to_page(NBPD); 2308 continue; 2309 } 2310 2311 /* We've found a valid PTE, so this page of PTEs has to go. */ 2312 if (pmap_pte_v(pte)) { 2313 /* Update statistics */ 2314 --pmap->pm_stats.resident_count; 2315 2316 /* 2317 * Add this page to our cache remove list, if we can. 2318 * If, however the cache remove list is totally full, 2319 * then do a complete cache invalidation taking note 2320 * to backtrack the PTE table beforehand, and ignore 2321 * the lists in future because there's no longer any 2322 * point in bothering with them (we've paid the 2323 * penalty, so will carry on unhindered). Otherwise, 2324 * when we fall out, we just clean the list. 2325 */ 2326 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2327 pa = pmap_pte_pa(pte); 2328 2329 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2330 /* Add to the clean list. */ 2331 cleanlist[cleanlist_idx].pte = pte; 2332 cleanlist[cleanlist_idx].va = sva; 2333 cleanlist_idx++; 2334 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2335 int cnt; 2336 2337 /* Nuke everything if needed. */ 2338 if (pmap_active) { 2339 cpu_idcache_wbinv_all(); 2340 cpu_tlb_flushID(); 2341 } 2342 2343 /* 2344 * Roll back the previous PTE list, 2345 * and zero out the current PTE. 2346 */ 2347 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2348 *cleanlist[cnt].pte = 0; 2349 pmap_pte_delref(pmap, cleanlist[cnt].va); 2350 } 2351 *pte = 0; 2352 pmap_pte_delref(pmap, sva); 2353 cleanlist_idx++; 2354 } else { 2355 /* 2356 * We've already nuked the cache and 2357 * TLB, so just carry on regardless, 2358 * and we won't need to do it again 2359 */ 2360 *pte = 0; 2361 pmap_pte_delref(pmap, sva); 2362 } 2363 2364 /* 2365 * Update flags. In a number of circumstances, 2366 * we could cluster a lot of these and do a 2367 * number of sequential pages in one go. 2368 */ 2369 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2370 struct pv_entry *pve; 2371 simple_lock(&pg->mdpage.pvh_slock); 2372 pve = pmap_remove_pv(pg, pmap, sva); 2373 pmap_free_pv(pmap, pve); 2374 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2375 simple_unlock(&pg->mdpage.pvh_slock); 2376 } 2377 } 2378 sva += NBPG; 2379 pte++; 2380 } 2381 2382 pmap_unmap_ptes(pmap); 2383 /* 2384 * Now, if we've fallen through down to here, chances are that there 2385 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2386 */ 2387 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2388 u_int cnt; 2389 2390 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2391 if (pmap_active) { 2392 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2393 NBPG); 2394 *cleanlist[cnt].pte = 0; 2395 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2396 } else 2397 *cleanlist[cnt].pte = 0; 2398 pmap_pte_delref(pmap, cleanlist[cnt].va); 2399 } 2400 } 2401 PMAP_MAP_TO_HEAD_UNLOCK(); 2402 } 2403 2404 /* 2405 * Routine: pmap_remove_all 2406 * Function: 2407 * Removes this physical page from 2408 * all physical maps in which it resides. 2409 * Reflects back modify bits to the pager. 2410 */ 2411 2412 static void 2413 pmap_remove_all(pg) 2414 struct vm_page *pg; 2415 { 2416 struct pv_entry *pv, *npv; 2417 struct pmap *pmap; 2418 pt_entry_t *pte, *ptes; 2419 2420 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2421 2422 /* set vm_page => pmap locking */ 2423 PMAP_HEAD_TO_MAP_LOCK(); 2424 2425 simple_lock(&pg->mdpage.pvh_slock); 2426 2427 pv = pg->mdpage.pvh_list; 2428 if (pv == NULL) { 2429 PDEBUG(0, printf("free page\n")); 2430 simple_unlock(&pg->mdpage.pvh_slock); 2431 PMAP_HEAD_TO_MAP_UNLOCK(); 2432 return; 2433 } 2434 pmap_clean_page(pv, FALSE); 2435 2436 while (pv) { 2437 pmap = pv->pv_pmap; 2438 ptes = pmap_map_ptes(pmap); 2439 pte = &ptes[arm_byte_to_page(pv->pv_va)]; 2440 2441 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2442 pv->pv_va, pv->pv_flags)); 2443 #ifdef DEBUG 2444 if (!pmap_pde_page(pmap_pde(pmap, pv->pv_va)) || 2445 !pmap_pte_v(pte) || pmap_pte_pa(pte) != pa) 2446 panic("pmap_remove_all: bad mapping"); 2447 #endif /* DEBUG */ 2448 2449 /* 2450 * Update statistics 2451 */ 2452 --pmap->pm_stats.resident_count; 2453 2454 /* Wired bit */ 2455 if (pv->pv_flags & PT_W) 2456 --pmap->pm_stats.wired_count; 2457 2458 /* 2459 * Invalidate the PTEs. 2460 * XXX: should cluster them up and invalidate as many 2461 * as possible at once. 2462 */ 2463 2464 #ifdef needednotdone 2465 reduce wiring count on page table pages as references drop 2466 #endif 2467 2468 *pte = 0; 2469 pmap_pte_delref(pmap, pv->pv_va); 2470 2471 npv = pv->pv_next; 2472 pmap_free_pv(pmap, pv); 2473 pv = npv; 2474 pmap_unmap_ptes(pmap); 2475 } 2476 pg->mdpage.pvh_list = NULL; 2477 simple_unlock(&pg->mdpage.pvh_slock); 2478 PMAP_HEAD_TO_MAP_UNLOCK(); 2479 2480 PDEBUG(0, printf("done\n")); 2481 cpu_tlb_flushID(); 2482 cpu_cpwait(); 2483 } 2484 2485 2486 /* 2487 * Set the physical protection on the specified range of this map as requested. 2488 */ 2489 2490 void 2491 pmap_protect(pmap, sva, eva, prot) 2492 struct pmap *pmap; 2493 vaddr_t sva; 2494 vaddr_t eva; 2495 vm_prot_t prot; 2496 { 2497 pt_entry_t *pte = NULL, *ptes; 2498 struct vm_page *pg; 2499 int armprot; 2500 int flush = 0; 2501 paddr_t pa; 2502 2503 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2504 pmap, sva, eva, prot)); 2505 2506 if (~prot & VM_PROT_READ) { 2507 /* Just remove the mappings. */ 2508 pmap_remove(pmap, sva, eva); 2509 /* pmap_update not needed as it should be called by the caller 2510 * of pmap_protect */ 2511 return; 2512 } 2513 if (prot & VM_PROT_WRITE) { 2514 /* 2515 * If this is a read->write transition, just ignore it and let 2516 * uvm_fault() take care of it later. 2517 */ 2518 return; 2519 } 2520 2521 sva &= PG_FRAME; 2522 eva &= PG_FRAME; 2523 2524 /* Need to lock map->head */ 2525 PMAP_MAP_TO_HEAD_LOCK(); 2526 2527 ptes = pmap_map_ptes(pmap); 2528 /* 2529 * We need to acquire a pointer to a page table page before entering 2530 * the following loop. 2531 */ 2532 while (sva < eva) { 2533 if (pmap_pde_page(pmap_pde(pmap, sva))) 2534 break; 2535 sva = (sva & PD_MASK) + NBPD; 2536 } 2537 2538 pte = &ptes[arm_byte_to_page(sva)]; 2539 2540 while (sva < eva) { 2541 /* only check once in a while */ 2542 if ((sva & PT_MASK) == 0) { 2543 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2544 /* We can race ahead here, to the next pde. */ 2545 sva += NBPD; 2546 pte += arm_byte_to_page(NBPD); 2547 continue; 2548 } 2549 } 2550 2551 if (!pmap_pte_v(pte)) 2552 goto next; 2553 2554 flush = 1; 2555 2556 armprot = 0; 2557 if (sva < VM_MAXUSER_ADDRESS) 2558 armprot |= PT_AP(AP_U); 2559 else if (sva < VM_MAX_ADDRESS) 2560 armprot |= PT_AP(AP_W); /* XXX Ekk what is this ? */ 2561 *pte = (*pte & 0xfffff00f) | armprot; 2562 2563 pa = pmap_pte_pa(pte); 2564 2565 /* Get the physical page index */ 2566 2567 /* Clear write flag */ 2568 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2569 simple_lock(&pg->mdpage.pvh_slock); 2570 (void) pmap_modify_pv(pmap, sva, pg, PT_Wr, 0); 2571 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2572 simple_unlock(&pg->mdpage.pvh_slock); 2573 } 2574 2575 next: 2576 sva += NBPG; 2577 pte++; 2578 } 2579 pmap_unmap_ptes(pmap); 2580 PMAP_MAP_TO_HEAD_UNLOCK(); 2581 if (flush) 2582 cpu_tlb_flushID(); 2583 } 2584 2585 /* 2586 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2587 * int flags) 2588 * 2589 * Insert the given physical page (p) at 2590 * the specified virtual address (v) in the 2591 * target physical map with the protection requested. 2592 * 2593 * If specified, the page will be wired down, meaning 2594 * that the related pte can not be reclaimed. 2595 * 2596 * NB: This is the only routine which MAY NOT lazy-evaluate 2597 * or lose information. That is, this routine must actually 2598 * insert this page into the given map NOW. 2599 */ 2600 2601 int 2602 pmap_enter(pmap, va, pa, prot, flags) 2603 struct pmap *pmap; 2604 vaddr_t va; 2605 paddr_t pa; 2606 vm_prot_t prot; 2607 int flags; 2608 { 2609 pt_entry_t *pte, *ptes; 2610 u_int npte; 2611 paddr_t opa; 2612 int nflags; 2613 boolean_t wired = (flags & PMAP_WIRED) != 0; 2614 struct vm_page *pg; 2615 struct pv_entry *pve; 2616 int error; 2617 2618 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2619 va, pa, pmap, prot, wired)); 2620 2621 #ifdef DIAGNOSTIC 2622 /* Valid address ? */ 2623 if (va >= (pmap_curmaxkvaddr)) 2624 panic("pmap_enter: too big"); 2625 if (pmap != pmap_kernel() && va != 0) { 2626 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2627 panic("pmap_enter: kernel page in user map"); 2628 } else { 2629 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2630 panic("pmap_enter: user page in kernel map"); 2631 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2632 panic("pmap_enter: entering PT page"); 2633 } 2634 #endif 2635 /* 2636 * Get a pointer to the page. Later on in this function, we 2637 * test for a managed page by checking pg != NULL. 2638 */ 2639 pg = PHYS_TO_VM_PAGE(pa); 2640 2641 /* get lock */ 2642 PMAP_MAP_TO_HEAD_LOCK(); 2643 /* 2644 * Get a pointer to the pte for this virtual address. If the 2645 * pte pointer is NULL then we are missing the L2 page table 2646 * so we need to create one. 2647 */ 2648 /* XXX horrible hack to get us working with lockdebug */ 2649 simple_lock(&pmap->pm_obj.vmobjlock); 2650 pte = pmap_pte(pmap, va); 2651 if (!pte) { 2652 struct vm_page *ptp; 2653 KASSERT(pmap != pmap_kernel()); /* kernel should have pre-grown */ 2654 2655 /* if failure is allowed then don't try too hard */ 2656 ptp = pmap_get_ptp(pmap, va, flags & PMAP_CANFAIL); 2657 if (ptp == NULL) { 2658 if (flags & PMAP_CANFAIL) { 2659 error = ENOMEM; 2660 goto out; 2661 } 2662 panic("pmap_enter: get ptp failed"); 2663 } 2664 2665 pte = pmap_pte(pmap, va); 2666 #ifdef DIAGNOSTIC 2667 if (!pte) 2668 panic("pmap_enter: no pte"); 2669 #endif 2670 } 2671 2672 nflags = 0; 2673 if (prot & VM_PROT_WRITE) 2674 nflags |= PT_Wr; 2675 if (wired) 2676 nflags |= PT_W; 2677 2678 /* More debugging info */ 2679 PDEBUG(5, printf("pmap_enter: pte for V%08lx = V%p (%08x)\n", va, pte, 2680 *pte)); 2681 2682 /* Is the pte valid ? If so then this page is already mapped */ 2683 if (pmap_pte_v(pte)) { 2684 /* Get the physical address of the current page mapped */ 2685 opa = pmap_pte_pa(pte); 2686 2687 #ifdef MYCROFT_HACK 2688 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx\n", pmap, va, pa, opa); 2689 #endif 2690 2691 /* Are we mapping the same page ? */ 2692 if (opa == pa) { 2693 /* All we must be doing is changing the protection */ 2694 PDEBUG(0, printf("Case 02 in pmap_enter (V%08lx P%08lx)\n", 2695 va, pa)); 2696 2697 /* Has the wiring changed ? */ 2698 if (pg != NULL) { 2699 simple_lock(&pg->mdpage.pvh_slock); 2700 (void) pmap_modify_pv(pmap, va, pg, 2701 PT_Wr | PT_W, nflags); 2702 simple_unlock(&pg->mdpage.pvh_slock); 2703 } 2704 } else { 2705 struct vm_page *opg; 2706 2707 /* We are replacing the page with a new one. */ 2708 cpu_idcache_wbinv_range(va, NBPG); 2709 2710 PDEBUG(0, printf("Case 03 in pmap_enter (V%08lx P%08lx P%08lx)\n", 2711 va, pa, opa)); 2712 2713 /* 2714 * If it is part of our managed memory then we 2715 * must remove it from the PV list 2716 */ 2717 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2718 simple_lock(&opg->mdpage.pvh_slock); 2719 pve = pmap_remove_pv(opg, pmap, va); 2720 simple_unlock(&opg->mdpage.pvh_slock); 2721 } else { 2722 pve = NULL; 2723 } 2724 2725 goto enter; 2726 } 2727 } else { 2728 opa = 0; 2729 pve = NULL; 2730 pmap_pte_addref(pmap, va); 2731 2732 /* pte is not valid so we must be hooking in a new page */ 2733 ++pmap->pm_stats.resident_count; 2734 2735 enter: 2736 /* 2737 * Enter on the PV list if part of our managed memory 2738 */ 2739 if (pmap_initialized && pg != NULL) { 2740 if (pve == NULL) { 2741 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2742 if (pve == NULL) { 2743 if (flags & PMAP_CANFAIL) { 2744 error = ENOMEM; 2745 goto out; 2746 } 2747 panic("pmap_enter: no pv entries available"); 2748 } 2749 } 2750 /* enter_pv locks pvh when adding */ 2751 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2752 } else { 2753 pg = NULL; 2754 if (pve != NULL) 2755 pmap_free_pv(pmap, pve); 2756 } 2757 } 2758 2759 #ifdef MYCROFT_HACK 2760 if (mycroft_hack) 2761 printf("pmap_enter: pmap=%p va=%lx pa=%lx opa=%lx bank=%d off=%d pv=%p\n", pmap, va, pa, opa, bank, off, pv); 2762 #endif 2763 2764 /* Construct the pte, giving the correct access. */ 2765 npte = (pa & PG_FRAME); 2766 2767 /* VA 0 is magic. */ 2768 if (pmap != pmap_kernel() && va != 0) 2769 npte |= PT_AP(AP_U); 2770 2771 if (pmap_initialized && pg != NULL) { 2772 #ifdef DIAGNOSTIC 2773 if ((flags & VM_PROT_ALL) & ~prot) 2774 panic("pmap_enter: access_type exceeds prot"); 2775 #endif 2776 npte |= pte_cache_mode; 2777 if (flags & VM_PROT_WRITE) { 2778 npte |= L2_SPAGE | PT_AP(AP_W); 2779 pg->mdpage.pvh_attrs |= PT_H | PT_M; 2780 } else if (flags & VM_PROT_ALL) { 2781 npte |= L2_SPAGE; 2782 pg->mdpage.pvh_attrs |= PT_H; 2783 } else 2784 npte |= L2_INVAL; 2785 } else { 2786 if (prot & VM_PROT_WRITE) 2787 npte |= L2_SPAGE | PT_AP(AP_W); 2788 else if (prot & VM_PROT_ALL) 2789 npte |= L2_SPAGE; 2790 else 2791 npte |= L2_INVAL; 2792 } 2793 2794 #ifdef MYCROFT_HACK 2795 if (mycroft_hack) 2796 printf("pmap_enter: pmap=%p va=%lx pa=%lx prot=%x wired=%d access_type=%x npte=%08x\n", pmap, va, pa, prot, wired, flags & VM_PROT_ALL, npte); 2797 #endif 2798 2799 *pte = npte; 2800 2801 if (pmap_initialized && pg != NULL) { 2802 boolean_t pmap_active = FALSE; 2803 /* XXX this will change once the whole of pmap_enter uses 2804 * map_ptes 2805 */ 2806 ptes = pmap_map_ptes(pmap); 2807 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) 2808 || (pmap == pmap_kernel())) 2809 pmap_active = TRUE; 2810 simple_lock(&pg->mdpage.pvh_slock); 2811 pmap_vac_me_harder(pmap, pg, ptes, pmap_active); 2812 simple_unlock(&pg->mdpage.pvh_slock); 2813 pmap_unmap_ptes(pmap); 2814 } 2815 2816 /* Better flush the TLB ... */ 2817 cpu_tlb_flushID_SE(va); 2818 error = 0; 2819 out: 2820 simple_unlock(&pmap->pm_obj.vmobjlock); 2821 PMAP_MAP_TO_HEAD_UNLOCK(); 2822 PDEBUG(5, printf("pmap_enter: pte = V%p %08x\n", pte, *pte)); 2823 2824 return error; 2825 } 2826 2827 /* 2828 * pmap_kenter_pa: enter a kernel mapping 2829 * 2830 * => no need to lock anything assume va is already allocated 2831 * => should be faster than normal pmap enter function 2832 */ 2833 void 2834 pmap_kenter_pa(va, pa, prot) 2835 vaddr_t va; 2836 paddr_t pa; 2837 vm_prot_t prot; 2838 { 2839 pt_entry_t *pte; 2840 2841 pte = vtopte(va); 2842 KASSERT(!pmap_pte_v(pte)); 2843 *pte = L2_PTE(pa, AP_KRW); 2844 } 2845 2846 void 2847 pmap_kremove(va, len) 2848 vaddr_t va; 2849 vsize_t len; 2850 { 2851 pt_entry_t *pte; 2852 2853 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2854 2855 /* 2856 * We assume that we will only be called with small 2857 * regions of memory. 2858 */ 2859 2860 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2861 pte = vtopte(va); 2862 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2863 *pte = 0; 2864 cpu_tlb_flushID_SE(va); 2865 } 2866 } 2867 2868 /* 2869 * pmap_page_protect: 2870 * 2871 * Lower the permission for all mappings to a given page. 2872 */ 2873 2874 void 2875 pmap_page_protect(pg, prot) 2876 struct vm_page *pg; 2877 vm_prot_t prot; 2878 { 2879 2880 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2881 VM_PAGE_TO_PHYS(pg), prot)); 2882 2883 switch(prot) { 2884 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2885 case VM_PROT_READ|VM_PROT_WRITE: 2886 return; 2887 2888 case VM_PROT_READ: 2889 case VM_PROT_READ|VM_PROT_EXECUTE: 2890 pmap_copy_on_write(pg); 2891 break; 2892 2893 default: 2894 pmap_remove_all(pg); 2895 break; 2896 } 2897 } 2898 2899 2900 /* 2901 * Routine: pmap_unwire 2902 * Function: Clear the wired attribute for a map/virtual-address 2903 * pair. 2904 * In/out conditions: 2905 * The mapping must already exist in the pmap. 2906 */ 2907 2908 void 2909 pmap_unwire(pmap, va) 2910 struct pmap *pmap; 2911 vaddr_t va; 2912 { 2913 pt_entry_t *pte; 2914 paddr_t pa; 2915 struct vm_page *pg; 2916 2917 /* 2918 * Make sure pmap is valid. -dct 2919 */ 2920 if (pmap == NULL) 2921 return; 2922 2923 /* Get the pte */ 2924 pte = pmap_pte(pmap, va); 2925 if (!pte) 2926 return; 2927 2928 /* Extract the physical address of the page */ 2929 pa = pmap_pte_pa(pte); 2930 2931 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2932 return; 2933 2934 simple_lock(&pg->mdpage.pvh_slock); 2935 /* Update the wired bit in the pv entry for this page. */ 2936 (void) pmap_modify_pv(pmap, va, pg, PT_W, 0); 2937 simple_unlock(&pg->mdpage.pvh_slock); 2938 } 2939 2940 /* 2941 * pt_entry_t *pmap_pte(struct pmap *pmap, vaddr_t va) 2942 * 2943 * Return the pointer to a page table entry corresponding to the supplied 2944 * virtual address. 2945 * 2946 * The page directory is first checked to make sure that a page table 2947 * for the address in question exists and if it does a pointer to the 2948 * entry is returned. 2949 * 2950 * The way this works is that that the kernel page tables are mapped 2951 * into the memory map at ALT_PAGE_TBLS_BASE to ALT_PAGE_TBLS_BASE+4MB. 2952 * This allows page tables to be located quickly. 2953 */ 2954 pt_entry_t * 2955 pmap_pte(pmap, va) 2956 struct pmap *pmap; 2957 vaddr_t va; 2958 { 2959 pt_entry_t *ptp; 2960 pt_entry_t *result; 2961 2962 /* The pmap must be valid */ 2963 if (!pmap) 2964 return(NULL); 2965 2966 /* Return the address of the pte */ 2967 PDEBUG(10, printf("pmap_pte: pmap=%p va=V%08lx pde = V%p (%08X)\n", 2968 pmap, va, pmap_pde(pmap, va), *(pmap_pde(pmap, va)))); 2969 2970 /* Do we have a valid pde ? If not we don't have a page table */ 2971 if (!pmap_pde_page(pmap_pde(pmap, va))) { 2972 PDEBUG(0, printf("pmap_pte: failed - pde = %p\n", 2973 pmap_pde(pmap, va))); 2974 return(NULL); 2975 } 2976 2977 PDEBUG(10, printf("pmap pagetable = P%08lx current = P%08x\n", 2978 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2979 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 2980 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME))); 2981 2982 /* 2983 * If the pmap is the kernel pmap or the pmap is the active one 2984 * then we can just return a pointer to entry relative to 2985 * PROCESS_PAGE_TBLS_BASE. 2986 * Otherwise we need to map the page tables to an alternative 2987 * address and reference them there. 2988 */ 2989 if (pmap == pmap_kernel() || pmap->pm_pptpt 2990 == (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 2991 + ((PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) & 2992 ~3) + (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 2993 ptp = (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 2994 } else { 2995 struct proc *p = curproc; 2996 2997 /* If we don't have a valid curproc use proc0 */ 2998 /* Perhaps we should just use kernel_pmap instead */ 2999 if (p == NULL) 3000 p = &proc0; 3001 #ifdef DIAGNOSTIC 3002 /* 3003 * The pmap should always be valid for the process so 3004 * panic if it is not. 3005 */ 3006 if (!p->p_vmspace || !p->p_vmspace->vm_map.pmap) { 3007 printf("pmap_pte: va=%08lx p=%p vm=%p\n", 3008 va, p, p->p_vmspace); 3009 console_debugger(); 3010 } 3011 /* 3012 * The pmap for the current process should be mapped. If it 3013 * is not then we have a problem. 3014 */ 3015 if (p->p_vmspace->vm_map.pmap->pm_pptpt != 3016 (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3017 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3018 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & PG_FRAME)) { 3019 printf("pmap pagetable = P%08lx current = P%08x ", 3020 pmap->pm_pptpt, (*((pt_entry_t *)(PROCESS_PAGE_TBLS_BASE 3021 + (PROCESS_PAGE_TBLS_BASE >> (PGSHIFT - 2)) + 3022 (PROCESS_PAGE_TBLS_BASE >> PDSHIFT))) & 3023 PG_FRAME)); 3024 printf("pptpt=%lx\n", p->p_vmspace->vm_map.pmap->pm_pptpt); 3025 panic("pmap_pte: current and pmap mismatch\n"); 3026 } 3027 #endif 3028 3029 ptp = (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3030 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3031 pmap->pm_pptpt, FALSE); 3032 cpu_tlb_flushD(); 3033 cpu_cpwait(); 3034 } 3035 PDEBUG(10, printf("page tables base = %p offset=%lx\n", ptp, 3036 ((va >> (PGSHIFT-2)) & ~3))); 3037 result = (pt_entry_t *)((char *)ptp + ((va >> (PGSHIFT-2)) & ~3)); 3038 return(result); 3039 } 3040 3041 /* 3042 * Routine: pmap_extract 3043 * Function: 3044 * Extract the physical page address associated 3045 * with the given map/virtual_address pair. 3046 */ 3047 boolean_t 3048 pmap_extract(pmap, va, pap) 3049 struct pmap *pmap; 3050 vaddr_t va; 3051 paddr_t *pap; 3052 { 3053 pd_entry_t *pde; 3054 pt_entry_t *pte, *ptes; 3055 paddr_t pa; 3056 boolean_t rv = TRUE; 3057 3058 PDEBUG(5, printf("pmap_extract: pmap=%p, va=V%08lx\n", pmap, va)); 3059 3060 /* 3061 * Get the pte for this virtual address. 3062 */ 3063 pde = pmap_pde(pmap, va); 3064 ptes = pmap_map_ptes(pmap); 3065 pte = &ptes[arm_byte_to_page(va)]; 3066 3067 if (pmap_pde_section(pde)) { 3068 pa = (*pde & PD_MASK) | (va & (L1_SEC_SIZE - 1)); 3069 goto out; 3070 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 3071 rv = FALSE; 3072 goto out; 3073 } 3074 3075 if ((*pte & L2_MASK) == L2_LPAGE) { 3076 /* Extract the physical address from the pte */ 3077 pa = *pte & ~(L2_LPAGE_SIZE - 1); 3078 3079 PDEBUG(5, printf("pmap_extract: LPAGE pa = P%08lx\n", 3080 (pa | (va & (L2_LPAGE_SIZE - 1))))); 3081 3082 if (pap != NULL) 3083 *pap = pa | (va & (L2_LPAGE_SIZE - 1)); 3084 goto out; 3085 } 3086 3087 /* Extract the physical address from the pte */ 3088 pa = pmap_pte_pa(pte); 3089 3090 PDEBUG(5, printf("pmap_extract: SPAGE pa = P%08lx\n", 3091 (pa | (va & ~PG_FRAME)))); 3092 3093 if (pap != NULL) 3094 *pap = pa | (va & ~PG_FRAME); 3095 out: 3096 pmap_unmap_ptes(pmap); 3097 return (rv); 3098 } 3099 3100 3101 /* 3102 * Copy the range specified by src_addr/len from the source map to the 3103 * range dst_addr/len in the destination map. 3104 * 3105 * This routine is only advisory and need not do anything. 3106 */ 3107 3108 void 3109 pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) 3110 struct pmap *dst_pmap; 3111 struct pmap *src_pmap; 3112 vaddr_t dst_addr; 3113 vsize_t len; 3114 vaddr_t src_addr; 3115 { 3116 PDEBUG(0, printf("pmap_copy(%p, %p, %lx, %lx, %lx)\n", 3117 dst_pmap, src_pmap, dst_addr, len, src_addr)); 3118 } 3119 3120 #if defined(PMAP_DEBUG) 3121 void 3122 pmap_dump_pvlist(phys, m) 3123 vaddr_t phys; 3124 char *m; 3125 { 3126 struct vm_page *pg; 3127 struct pv_entry *pv; 3128 3129 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 3130 printf("INVALID PA\n"); 3131 return; 3132 } 3133 simple_lock(&pg->mdpage.pvh_slock); 3134 printf("%s %08lx:", m, phys); 3135 if (pg->mdpage.pvh_list == NULL) { 3136 printf(" no mappings\n"); 3137 return; 3138 } 3139 3140 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 3141 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 3142 pv->pv_va, pv->pv_flags); 3143 3144 printf("\n"); 3145 simple_unlock(&pg->mdpage.pvh_slock); 3146 } 3147 3148 #endif /* PMAP_DEBUG */ 3149 3150 static pt_entry_t * 3151 pmap_map_ptes(struct pmap *pmap) 3152 { 3153 struct proc *p; 3154 3155 /* the kernel's pmap is always accessible */ 3156 if (pmap == pmap_kernel()) { 3157 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE ; 3158 } 3159 3160 if (pmap_is_curpmap(pmap)) { 3161 simple_lock(&pmap->pm_obj.vmobjlock); 3162 return (pt_entry_t *)PROCESS_PAGE_TBLS_BASE; 3163 } 3164 3165 p = curproc; 3166 3167 if (p == NULL) 3168 p = &proc0; 3169 3170 /* need to lock both curpmap and pmap: use ordered locking */ 3171 if ((unsigned) pmap < (unsigned) curproc->p_vmspace->vm_map.pmap) { 3172 simple_lock(&pmap->pm_obj.vmobjlock); 3173 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3174 } else { 3175 simple_lock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3176 simple_lock(&pmap->pm_obj.vmobjlock); 3177 } 3178 3179 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, ALT_PAGE_TBLS_BASE, 3180 pmap->pm_pptpt, FALSE); 3181 cpu_tlb_flushD(); 3182 cpu_cpwait(); 3183 return (pt_entry_t *)ALT_PAGE_TBLS_BASE; 3184 } 3185 3186 /* 3187 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3188 */ 3189 3190 static void 3191 pmap_unmap_ptes(pmap) 3192 struct pmap *pmap; 3193 { 3194 if (pmap == pmap_kernel()) { 3195 return; 3196 } 3197 if (pmap_is_curpmap(pmap)) { 3198 simple_unlock(&pmap->pm_obj.vmobjlock); 3199 } else { 3200 simple_unlock(&pmap->pm_obj.vmobjlock); 3201 simple_unlock(&curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3202 } 3203 } 3204 3205 /* 3206 * Modify pte bits for all ptes corresponding to the given physical address. 3207 * We use `maskbits' rather than `clearbits' because we're always passing 3208 * constants and the latter would require an extra inversion at run-time. 3209 */ 3210 3211 static void 3212 pmap_clearbit(pg, maskbits) 3213 struct vm_page *pg; 3214 unsigned int maskbits; 3215 { 3216 struct pv_entry *pv; 3217 pt_entry_t *pte; 3218 vaddr_t va; 3219 int tlbentry; 3220 3221 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3222 VM_PAGE_TO_PHYS(pg), maskbits)); 3223 3224 tlbentry = 0; 3225 3226 PMAP_HEAD_TO_MAP_LOCK(); 3227 simple_lock(&pg->mdpage.pvh_slock); 3228 3229 /* 3230 * Clear saved attributes (modify, reference) 3231 */ 3232 pg->mdpage.pvh_attrs &= ~maskbits; 3233 3234 if (pg->mdpage.pvh_list == NULL) { 3235 simple_unlock(&pg->mdpage.pvh_slock); 3236 PMAP_HEAD_TO_MAP_UNLOCK(); 3237 return; 3238 } 3239 3240 /* 3241 * Loop over all current mappings setting/clearing as appropos 3242 */ 3243 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3244 va = pv->pv_va; 3245 pv->pv_flags &= ~maskbits; 3246 pte = pmap_pte(pv->pv_pmap, va); 3247 KASSERT(pte != NULL); 3248 if (maskbits & (PT_Wr|PT_M)) { 3249 if ((pv->pv_flags & PT_NC)) { 3250 /* 3251 * Entry is not cacheable: reenable 3252 * the cache, nothing to flush 3253 * 3254 * Don't turn caching on again if this 3255 * is a modified emulation. This 3256 * would be inconsitent with the 3257 * settings created by 3258 * pmap_vac_me_harder(). 3259 * 3260 * There's no need to call 3261 * pmap_vac_me_harder() here: all 3262 * pages are loosing their write 3263 * permission. 3264 * 3265 */ 3266 if (maskbits & PT_Wr) { 3267 *pte |= pte_cache_mode; 3268 pv->pv_flags &= ~PT_NC; 3269 } 3270 } else if (pmap_is_curpmap(pv->pv_pmap)) 3271 /* 3272 * Entry is cacheable: check if pmap is 3273 * current if it is flush it, 3274 * otherwise it won't be in the cache 3275 */ 3276 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3277 3278 /* make the pte read only */ 3279 *pte &= ~PT_AP(AP_W); 3280 } 3281 3282 if (maskbits & PT_H) 3283 *pte = (*pte & ~L2_MASK) | L2_INVAL; 3284 3285 if (pmap_is_curpmap(pv->pv_pmap)) 3286 /* 3287 * if we had cacheable pte's we'd clean the 3288 * pte out to memory here 3289 * 3290 * flush tlb entry as it's in the current pmap 3291 */ 3292 cpu_tlb_flushID_SE(pv->pv_va); 3293 } 3294 cpu_cpwait(); 3295 3296 simple_unlock(&pg->mdpage.pvh_slock); 3297 PMAP_HEAD_TO_MAP_UNLOCK(); 3298 } 3299 3300 /* 3301 * pmap_clear_modify: 3302 * 3303 * Clear the "modified" attribute for a page. 3304 */ 3305 boolean_t 3306 pmap_clear_modify(pg) 3307 struct vm_page *pg; 3308 { 3309 boolean_t rv; 3310 3311 if (pg->mdpage.pvh_attrs & PT_M) { 3312 rv = TRUE; 3313 pmap_clearbit(pg, PT_M); 3314 } else 3315 rv = FALSE; 3316 3317 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3318 VM_PAGE_TO_PHYS(pg), rv)); 3319 3320 return (rv); 3321 } 3322 3323 /* 3324 * pmap_clear_reference: 3325 * 3326 * Clear the "referenced" attribute for a page. 3327 */ 3328 boolean_t 3329 pmap_clear_reference(pg) 3330 struct vm_page *pg; 3331 { 3332 boolean_t rv; 3333 3334 if (pg->mdpage.pvh_attrs & PT_H) { 3335 rv = TRUE; 3336 pmap_clearbit(pg, PT_H); 3337 } else 3338 rv = FALSE; 3339 3340 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3341 VM_PAGE_TO_PHYS(pg), rv)); 3342 3343 return (rv); 3344 } 3345 3346 3347 void 3348 pmap_copy_on_write(pg) 3349 struct vm_page *pg; 3350 { 3351 PDEBUG(0, printf("pmap_copy_on_write pa=%08lx\n", VM_PAGE_TO_PHYS(pg))); 3352 pmap_clearbit(pg, PT_Wr); 3353 } 3354 3355 /* 3356 * pmap_is_modified: 3357 * 3358 * Test if a page has the "modified" attribute. 3359 */ 3360 /* See <arm/arm32/pmap.h> */ 3361 3362 /* 3363 * pmap_is_referenced: 3364 * 3365 * Test if a page has the "referenced" attribute. 3366 */ 3367 /* See <arm/arm32/pmap.h> */ 3368 3369 int 3370 pmap_modified_emulation(pmap, va) 3371 struct pmap *pmap; 3372 vaddr_t va; 3373 { 3374 pt_entry_t *pte; 3375 paddr_t pa; 3376 struct vm_page *pg; 3377 u_int flags; 3378 3379 PDEBUG(2, printf("pmap_modified_emulation\n")); 3380 3381 /* Get the pte */ 3382 pte = pmap_pte(pmap, va); 3383 if (!pte) { 3384 PDEBUG(2, printf("no pte\n")); 3385 return(0); 3386 } 3387 3388 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3389 3390 /* Check for a zero pte */ 3391 if (*pte == 0) 3392 return(0); 3393 3394 /* This can happen if user code tries to access kernel memory. */ 3395 if ((*pte & PT_AP(AP_W)) != 0) 3396 return (0); 3397 3398 /* Extract the physical address of the page */ 3399 pa = pmap_pte_pa(pte); 3400 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3401 return(0); 3402 3403 /* Get the current flags for this page. */ 3404 PMAP_HEAD_TO_MAP_LOCK(); 3405 simple_lock(&pg->mdpage.pvh_slock); 3406 3407 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3408 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3409 3410 /* 3411 * Do the flags say this page is writable ? If not then it is a 3412 * genuine write fault. If yes then the write fault is our fault 3413 * as we did not reflect the write access in the PTE. Now we know 3414 * a write has occurred we can correct this and also set the 3415 * modified bit 3416 */ 3417 if (~flags & PT_Wr) { 3418 simple_unlock(&pg->mdpage.pvh_slock); 3419 PMAP_HEAD_TO_MAP_UNLOCK(); 3420 return(0); 3421 } 3422 3423 PDEBUG(0, printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %p (%08x)\n", 3424 va, pte, *pte)); 3425 pg->mdpage.pvh_attrs |= PT_H | PT_M; 3426 3427 /* 3428 * Re-enable write permissions for the page. No need to call 3429 * pmap_vac_me_harder(), since this is just a 3430 * modified-emulation fault, and the PT_Wr bit isn't changing. We've 3431 * already set the cacheable bits based on the assumption that we 3432 * can write to this page. 3433 */ 3434 *pte = (*pte & ~L2_MASK) | L2_SPAGE | PT_AP(AP_W); 3435 PDEBUG(0, printf("->(%08x)\n", *pte)); 3436 3437 simple_unlock(&pg->mdpage.pvh_slock); 3438 PMAP_HEAD_TO_MAP_UNLOCK(); 3439 /* Return, indicating the problem has been dealt with */ 3440 cpu_tlb_flushID_SE(va); 3441 cpu_cpwait(); 3442 return(1); 3443 } 3444 3445 3446 int 3447 pmap_handled_emulation(pmap, va) 3448 struct pmap *pmap; 3449 vaddr_t va; 3450 { 3451 pt_entry_t *pte; 3452 paddr_t pa; 3453 struct vm_page *pg; 3454 3455 PDEBUG(2, printf("pmap_handled_emulation\n")); 3456 3457 /* Get the pte */ 3458 pte = pmap_pte(pmap, va); 3459 if (!pte) { 3460 PDEBUG(2, printf("no pte\n")); 3461 return(0); 3462 } 3463 3464 PDEBUG(1, printf("*pte=%08x\n", *pte)); 3465 3466 /* Check for a zero pte */ 3467 if (*pte == 0) 3468 return(0); 3469 3470 /* This can happen if user code tries to access kernel memory. */ 3471 if ((*pte & L2_MASK) != L2_INVAL) 3472 return (0); 3473 3474 /* Extract the physical address of the page */ 3475 pa = pmap_pte_pa(pte); 3476 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3477 return (0); 3478 3479 /* 3480 * Ok we just enable the pte and mark the attibs as handled 3481 */ 3482 PDEBUG(0, printf("pmap_handled_emulation: Got a hit va=%08lx pte = %p (%08x)\n", 3483 va, pte, *pte)); 3484 pg->mdpage.pvh_attrs |= PT_H; 3485 *pte = (*pte & ~L2_MASK) | L2_SPAGE; 3486 PDEBUG(0, printf("->(%08x)\n", *pte)); 3487 3488 /* Return, indicating the problem has been dealt with */ 3489 cpu_tlb_flushID_SE(va); 3490 cpu_cpwait(); 3491 return(1); 3492 } 3493 3494 3495 3496 3497 /* 3498 * pmap_collect: free resources held by a pmap 3499 * 3500 * => optional function. 3501 * => called when a process is swapped out to free memory. 3502 */ 3503 3504 void 3505 pmap_collect(pmap) 3506 struct pmap *pmap; 3507 { 3508 } 3509 3510 /* 3511 * Routine: pmap_procwr 3512 * 3513 * Function: 3514 * Synchronize caches corresponding to [addr, addr+len) in p. 3515 * 3516 */ 3517 void 3518 pmap_procwr(p, va, len) 3519 struct proc *p; 3520 vaddr_t va; 3521 int len; 3522 { 3523 /* We only need to do anything if it is the current process. */ 3524 if (p == curproc) 3525 cpu_icache_sync_range(va, len); 3526 } 3527 /* 3528 * PTP functions 3529 */ 3530 3531 /* 3532 * pmap_steal_ptp: Steal a PTP from somewhere else. 3533 * 3534 * This is just a placeholder, for now we never steal. 3535 */ 3536 3537 static struct vm_page * 3538 pmap_steal_ptp(struct pmap *pmap, vaddr_t va) 3539 { 3540 return (NULL); 3541 } 3542 3543 /* 3544 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3545 * 3546 * => pmap should NOT be pmap_kernel() 3547 * => pmap should be locked 3548 */ 3549 3550 static struct vm_page * 3551 pmap_get_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3552 { 3553 struct vm_page *ptp; 3554 3555 if (pmap_pde_page(pmap_pde(pmap, va))) { 3556 3557 /* valid... check hint (saves us a PA->PG lookup) */ 3558 #if 0 3559 if (pmap->pm_ptphint && 3560 ((unsigned)pmap_pde(pmap, va) & PG_FRAME) == 3561 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3562 return (pmap->pm_ptphint); 3563 #endif 3564 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3565 #ifdef DIAGNOSTIC 3566 if (ptp == NULL) 3567 panic("pmap_get_ptp: unmanaged user PTP"); 3568 #endif 3569 // pmap->pm_ptphint = ptp; 3570 return(ptp); 3571 } 3572 3573 /* allocate a new PTP (updates ptphint) */ 3574 return(pmap_alloc_ptp(pmap, va, just_try)); 3575 } 3576 3577 /* 3578 * pmap_alloc_ptp: allocate a PTP for a PMAP 3579 * 3580 * => pmap should already be locked by caller 3581 * => we use the ptp's wire_count to count the number of active mappings 3582 * in the PTP (we start it at one to prevent any chance this PTP 3583 * will ever leak onto the active/inactive queues) 3584 */ 3585 3586 /*__inline */ static struct vm_page * 3587 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va, boolean_t just_try) 3588 { 3589 struct vm_page *ptp; 3590 3591 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3592 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3593 if (ptp == NULL) { 3594 if (just_try) 3595 return (NULL); 3596 3597 ptp = pmap_steal_ptp(pmap, va); 3598 3599 if (ptp == NULL) 3600 return (NULL); 3601 /* Stole a page, zero it. */ 3602 pmap_zero_page(VM_PAGE_TO_PHYS(ptp)); 3603 } 3604 3605 /* got one! */ 3606 ptp->flags &= ~PG_BUSY; /* never busy */ 3607 ptp->wire_count = 1; /* no mappings yet */ 3608 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3609 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3610 // pmap->pm_ptphint = ptp; 3611 return (ptp); 3612 } 3613 3614 vaddr_t 3615 pmap_growkernel(maxkvaddr) 3616 vaddr_t maxkvaddr; 3617 { 3618 struct pmap *kpm = pmap_kernel(), *pm; 3619 int s; 3620 paddr_t ptaddr; 3621 struct vm_page *ptp; 3622 3623 if (maxkvaddr <= pmap_curmaxkvaddr) 3624 goto out; /* we are OK */ 3625 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3626 pmap_curmaxkvaddr, maxkvaddr)); 3627 3628 /* 3629 * whoops! we need to add kernel PTPs 3630 */ 3631 3632 s = splhigh(); /* to be safe */ 3633 simple_lock(&kpm->pm_obj.vmobjlock); 3634 /* due to the way the arm pmap works we map 4MB at a time */ 3635 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr ; pmap_curmaxkvaddr += 4 * NBPD) { 3636 3637 if (uvm.page_init_done == FALSE) { 3638 3639 /* 3640 * we're growing the kernel pmap early (from 3641 * uvm_pageboot_alloc()). this case must be 3642 * handled a little differently. 3643 */ 3644 3645 if (uvm_page_physget(&ptaddr) == FALSE) 3646 panic("pmap_growkernel: out of memory"); 3647 pmap_zero_page(ptaddr); 3648 3649 /* map this page in */ 3650 pmap_map_in_l1(kpm, (pmap_curmaxkvaddr + 1), ptaddr, TRUE); 3651 3652 /* count PTP as resident */ 3653 kpm->pm_stats.resident_count++; 3654 continue; 3655 } 3656 3657 /* 3658 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3659 * pmap_initialized == FALSE CASE! WE MAY BE 3660 * INVOKED WHILE pmap_init() IS RUNNING! 3661 */ 3662 3663 if ((ptp = pmap_alloc_ptp(kpm, (pmap_curmaxkvaddr + 1), FALSE)) == NULL) { 3664 panic("pmap_growkernel: alloc ptp failed"); 3665 } 3666 3667 /* distribute new kernel PTP to all active pmaps */ 3668 simple_lock(&pmaps_lock); 3669 LIST_FOREACH(pm, &pmaps, pm_list) { 3670 pmap_map_in_l1(pm, (pmap_curmaxkvaddr + 1), VM_PAGE_TO_PHYS(ptp), TRUE); 3671 } 3672 3673 simple_unlock(&pmaps_lock); 3674 } 3675 3676 /* 3677 * flush out the cache, expensive but growkernel will happen so 3678 * rarely 3679 */ 3680 cpu_tlb_flushD(); 3681 cpu_cpwait(); 3682 3683 simple_unlock(&kpm->pm_obj.vmobjlock); 3684 splx(s); 3685 3686 out: 3687 return (pmap_curmaxkvaddr); 3688 } 3689 3690 3691 3692 /************************ Bootstrapping routines ****************************/ 3693 3694 /* 3695 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3696 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3697 * find them as necessary. 3698 * 3699 * Note that the data on this list is not valid after initarm() returns. 3700 */ 3701 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3702 3703 static vaddr_t 3704 kernel_pt_lookup(paddr_t pa) 3705 { 3706 pv_addr_t *pv; 3707 3708 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3709 if (pv->pv_pa == pa) 3710 return (pv->pv_va); 3711 } 3712 return (0); 3713 } 3714 3715 /* 3716 * pmap_map_section: 3717 * 3718 * Create a single section mapping. 3719 */ 3720 void 3721 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3722 { 3723 pd_entry_t *pde = (pd_entry_t *) l1pt; 3724 pd_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3725 pd_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3726 3727 KASSERT(((va | pa) & (L1_SEC_SIZE - 1)) == 0); 3728 3729 pde[va >> PDSHIFT] = L1_SECPTE(pa & PD_MASK, ap, fl); 3730 } 3731 3732 /* 3733 * pmap_map_entry: 3734 * 3735 * Create a single page mapping. 3736 */ 3737 void 3738 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3739 { 3740 pd_entry_t *pde = (pd_entry_t *) l1pt; 3741 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3742 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3743 pt_entry_t *pte; 3744 3745 KASSERT(((va | pa) & PGOFSET) == 0); 3746 3747 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3748 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3749 3750 pte = (pt_entry_t *) 3751 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3752 if (pte == NULL) 3753 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3754 3755 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa & PG_FRAME, ap, fl); 3756 } 3757 3758 /* 3759 * pmap_link_l2pt: 3760 * 3761 * Link the L2 page table specified by "pa" into the L1 3762 * page table at the slot for "va". 3763 */ 3764 void 3765 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3766 { 3767 pd_entry_t *pde = (pd_entry_t *) l1pt; 3768 u_int slot = va >> PDSHIFT; 3769 3770 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3771 3772 pde[slot + 0] = L1_PTE(l2pv->pv_pa + 0x000); 3773 pde[slot + 1] = L1_PTE(l2pv->pv_pa + 0x400); 3774 pde[slot + 2] = L1_PTE(l2pv->pv_pa + 0x800); 3775 pde[slot + 3] = L1_PTE(l2pv->pv_pa + 0xc00); 3776 3777 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3778 } 3779 3780 /* 3781 * pmap_map_chunk: 3782 * 3783 * Map a chunk of memory using the most efficient mappings 3784 * possible (section, large page, small page) into the 3785 * provided L1 and L2 tables at the specified virtual address. 3786 */ 3787 vsize_t 3788 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3789 int prot, int cache) 3790 { 3791 pd_entry_t *pde = (pd_entry_t *) l1pt; 3792 pt_entry_t ap = (prot & VM_PROT_WRITE) ? AP_KRW : AP_KR; 3793 pt_entry_t fl = (cache == PTE_CACHE) ? pte_cache_mode : 0; 3794 pt_entry_t *pte; 3795 vsize_t resid; 3796 int i; 3797 3798 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3799 3800 if (l1pt == 0) 3801 panic("pmap_map_chunk: no L1 table provided"); 3802 3803 #ifdef VERBOSE_INIT_ARM 3804 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3805 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3806 #endif 3807 3808 size = resid; 3809 3810 while (resid > 0) { 3811 /* See if we can use a section mapping. */ 3812 if (((pa | va) & (L1_SEC_SIZE - 1)) == 0 && 3813 resid >= L1_SEC_SIZE) { 3814 #ifdef VERBOSE_INIT_ARM 3815 printf("S"); 3816 #endif 3817 pde[va >> PDSHIFT] = L1_SECPTE(pa, ap, fl); 3818 va += L1_SEC_SIZE; 3819 pa += L1_SEC_SIZE; 3820 resid -= L1_SEC_SIZE; 3821 continue; 3822 } 3823 3824 /* 3825 * Ok, we're going to use an L2 table. Make sure 3826 * one is actually in the corresponding L1 slot 3827 * for the current VA. 3828 */ 3829 if ((pde[va >> PDSHIFT] & L1_MASK) != L1_PAGE) 3830 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3831 3832 pte = (pt_entry_t *) 3833 kernel_pt_lookup(pde[va >> PDSHIFT] & PG_FRAME); 3834 if (pte == NULL) 3835 panic("pmap_map_chunk: can't find L2 table for VA" 3836 "0x%08lx", va); 3837 3838 /* See if we can use a L2 large page mapping. */ 3839 if (((pa | va) & (L2_LPAGE_SIZE - 1)) == 0 && 3840 resid >= L2_LPAGE_SIZE) { 3841 #ifdef VERBOSE_INIT_ARM 3842 printf("L"); 3843 #endif 3844 for (i = 0; i < 16; i++) { 3845 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3846 L2_LPTE(pa, ap, fl); 3847 } 3848 va += L2_LPAGE_SIZE; 3849 pa += L2_LPAGE_SIZE; 3850 resid -= L2_LPAGE_SIZE; 3851 continue; 3852 } 3853 3854 /* Use a small page mapping. */ 3855 #ifdef VERBOSE_INIT_ARM 3856 printf("P"); 3857 #endif 3858 pte[(va >> PGSHIFT) & 0x3ff] = L2_SPTE(pa, ap, fl); 3859 va += NBPG; 3860 pa += NBPG; 3861 resid -= NBPG; 3862 } 3863 #ifdef VERBOSE_INIT_ARM 3864 printf("\n"); 3865 #endif 3866 return (size); 3867 } 3868
Indexes created Mon Sep 29 13:09:55 GMT 2025