pmap.c revision 1.89
1 /* $NetBSD: pmap.c,v 1.89 2002/04/10 01:30:42 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 2002 Wasabi Systems, Inc. 5 * Copyright (c) 2001 Richard Earnshaw 6 * Copyright (c) 2001 Christopher Gilbert 7 * All rights reserved. 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the company nor the name of the author may be used to 15 * endorse or promote products derived from this software without specific 16 * prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 19 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 20 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 22 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1999 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Charles M. Hannum. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1994-1998 Mark Brinicombe. 69 * Copyright (c) 1994 Brini. 70 * All rights reserved. 71 * 72 * This code is derived from software written for Brini by Mark Brinicombe 73 * 74 * Redistribution and use in source and binary forms, with or without 75 * modification, are permitted provided that the following conditions 76 * are met: 77 * 1. Redistributions of source code must retain the above copyright 78 * notice, this list of conditions and the following disclaimer. 79 * 2. Redistributions in binary form must reproduce the above copyright 80 * notice, this list of conditions and the following disclaimer in the 81 * documentation and/or other materials provided with the distribution. 82 * 3. All advertising materials mentioning features or use of this software 83 * must display the following acknowledgement: 84 * This product includes software developed by Mark Brinicombe. 85 * 4. The name of the author may not be used to endorse or promote products 86 * derived from this software without specific prior written permission. 87 * 88 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 89 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 90 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 91 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 92 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 93 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 94 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 95 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 96 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 97 * 98 * RiscBSD kernel project 99 * 100 * pmap.c 101 * 102 * Machine dependant vm stuff 103 * 104 * Created : 20/09/94 105 */ 106 107 /* 108 * Performance improvements, UVM changes, overhauls and part-rewrites 109 * were contributed by Neil A. Carson <neil (at) causality.com>. 110 */ 111 112 /* 113 * The dram block info is currently referenced from the bootconfig. 114 * This should be placed in a separate structure. 115 */ 116 117 /* 118 * Special compilation symbols 119 * PMAP_DEBUG - Build in pmap_debug_level code 120 */ 121 122 /* Include header files */ 123 124 #include "opt_pmap_debug.h" 125 #include "opt_ddb.h" 126 127 #include <sys/types.h> 128 #include <sys/param.h> 129 #include <sys/kernel.h> 130 #include <sys/systm.h> 131 #include <sys/proc.h> 132 #include <sys/malloc.h> 133 #include <sys/user.h> 134 #include <sys/pool.h> 135 #include <sys/cdefs.h> 136 137 #include <uvm/uvm.h> 138 139 #include <machine/bootconfig.h> 140 #include <machine/bus.h> 141 #include <machine/pmap.h> 142 #include <machine/pcb.h> 143 #include <machine/param.h> 144 #include <arm/arm32/katelib.h> 145 146 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.89 2002/04/10 01:30:42 thorpej Exp $"); 147 #ifdef PMAP_DEBUG 148 #define PDEBUG(_lev_,_stat_) \ 149 if (pmap_debug_level >= (_lev_)) \ 150 ((_stat_)) 151 int pmap_debug_level = -2; 152 void pmap_dump_pvlist(vaddr_t phys, char *m); 153 154 /* 155 * for switching to potentially finer grained debugging 156 */ 157 #define PDB_FOLLOW 0x0001 158 #define PDB_INIT 0x0002 159 #define PDB_ENTER 0x0004 160 #define PDB_REMOVE 0x0008 161 #define PDB_CREATE 0x0010 162 #define PDB_PTPAGE 0x0020 163 #define PDB_GROWKERN 0x0040 164 #define PDB_BITS 0x0080 165 #define PDB_COLLECT 0x0100 166 #define PDB_PROTECT 0x0200 167 #define PDB_MAP_L1 0x0400 168 #define PDB_BOOTSTRAP 0x1000 169 #define PDB_PARANOIA 0x2000 170 #define PDB_WIRING 0x4000 171 #define PDB_PVDUMP 0x8000 172 173 int debugmap = 0; 174 int pmapdebug = PDB_PARANOIA | PDB_FOLLOW; 175 #define NPDEBUG(_lev_,_stat_) \ 176 if (pmapdebug & (_lev_)) \ 177 ((_stat_)) 178 179 #else /* PMAP_DEBUG */ 180 #define PDEBUG(_lev_,_stat_) /* Nothing */ 181 #define NPDEBUG(_lev_,_stat_) /* Nothing */ 182 #endif /* PMAP_DEBUG */ 183 184 struct pmap kernel_pmap_store; 185 186 /* 187 * linked list of all non-kernel pmaps 188 */ 189 190 static LIST_HEAD(, pmap) pmaps; 191 192 /* 193 * pool that pmap structures are allocated from 194 */ 195 196 struct pool pmap_pmap_pool; 197 198 static pt_entry_t *csrc_pte, *cdst_pte; 199 static vaddr_t csrcp, cdstp; 200 201 char *memhook; 202 extern caddr_t msgbufaddr; 203 204 boolean_t pmap_initialized = FALSE; /* Has pmap_init completed? */ 205 /* 206 * locking data structures 207 */ 208 209 static struct lock pmap_main_lock; 210 static struct simplelock pvalloc_lock; 211 static struct simplelock pmaps_lock; 212 #ifdef LOCKDEBUG 213 #define PMAP_MAP_TO_HEAD_LOCK() \ 214 (void) spinlockmgr(&pmap_main_lock, LK_SHARED, NULL) 215 #define PMAP_MAP_TO_HEAD_UNLOCK() \ 216 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 217 218 #define PMAP_HEAD_TO_MAP_LOCK() \ 219 (void) spinlockmgr(&pmap_main_lock, LK_EXCLUSIVE, NULL) 220 #define PMAP_HEAD_TO_MAP_UNLOCK() \ 221 (void) spinlockmgr(&pmap_main_lock, LK_RELEASE, NULL) 222 #else 223 #define PMAP_MAP_TO_HEAD_LOCK() /* nothing */ 224 #define PMAP_MAP_TO_HEAD_UNLOCK() /* nothing */ 225 #define PMAP_HEAD_TO_MAP_LOCK() /* nothing */ 226 #define PMAP_HEAD_TO_MAP_UNLOCK() /* nothing */ 227 #endif /* LOCKDEBUG */ 228 229 /* 230 * pv_page management structures: locked by pvalloc_lock 231 */ 232 233 TAILQ_HEAD(pv_pagelist, pv_page); 234 static struct pv_pagelist pv_freepages; /* list of pv_pages with free entrys */ 235 static struct pv_pagelist pv_unusedpgs; /* list of unused pv_pages */ 236 static int pv_nfpvents; /* # of free pv entries */ 237 static struct pv_page *pv_initpage; /* bootstrap page from kernel_map */ 238 static vaddr_t pv_cachedva; /* cached VA for later use */ 239 240 #define PVE_LOWAT (PVE_PER_PVPAGE / 2) /* free pv_entry low water mark */ 241 #define PVE_HIWAT (PVE_LOWAT + (PVE_PER_PVPAGE * 2)) 242 /* high water mark */ 243 244 /* 245 * local prototypes 246 */ 247 248 static struct pv_entry *pmap_add_pvpage __P((struct pv_page *, boolean_t)); 249 static struct pv_entry *pmap_alloc_pv __P((struct pmap *, int)); /* see codes below */ 250 #define ALLOCPV_NEED 0 /* need PV now */ 251 #define ALLOCPV_TRY 1 /* just try to allocate, don't steal */ 252 #define ALLOCPV_NONEED 2 /* don't need PV, just growing cache */ 253 static struct pv_entry *pmap_alloc_pvpage __P((struct pmap *, int)); 254 static void pmap_enter_pv __P((struct vm_page *, 255 struct pv_entry *, struct pmap *, 256 vaddr_t, struct vm_page *, int)); 257 static void pmap_free_pv __P((struct pmap *, struct pv_entry *)); 258 static void pmap_free_pvs __P((struct pmap *, struct pv_entry *)); 259 static void pmap_free_pv_doit __P((struct pv_entry *)); 260 static void pmap_free_pvpage __P((void)); 261 static boolean_t pmap_is_curpmap __P((struct pmap *)); 262 static struct pv_entry *pmap_remove_pv __P((struct vm_page *, struct pmap *, 263 vaddr_t)); 264 #define PMAP_REMOVE_ALL 0 /* remove all mappings */ 265 #define PMAP_REMOVE_SKIPWIRED 1 /* skip wired mappings */ 266 267 static u_int pmap_modify_pv __P((struct pmap *, vaddr_t, struct vm_page *, 268 u_int, u_int)); 269 270 /* 271 * Structure that describes and L1 table. 272 */ 273 struct l1pt { 274 SIMPLEQ_ENTRY(l1pt) pt_queue; /* Queue pointers */ 275 struct pglist pt_plist; /* Allocated page list */ 276 vaddr_t pt_va; /* Allocated virtual address */ 277 int pt_flags; /* Flags */ 278 }; 279 #define PTFLAG_STATIC 0x01 /* Statically allocated */ 280 #define PTFLAG_KPT 0x02 /* Kernel pt's are mapped */ 281 #define PTFLAG_CLEAN 0x04 /* L1 is clean */ 282 283 static void pmap_free_l1pt __P((struct l1pt *)); 284 static int pmap_allocpagedir __P((struct pmap *)); 285 static int pmap_clean_page __P((struct pv_entry *, boolean_t)); 286 static void pmap_remove_all __P((struct vm_page *)); 287 288 static struct vm_page *pmap_alloc_ptp __P((struct pmap *, vaddr_t)); 289 static struct vm_page *pmap_get_ptp __P((struct pmap *, vaddr_t)); 290 __inline static void pmap_clearbit __P((struct vm_page *, unsigned int)); 291 292 extern paddr_t physical_start; 293 extern paddr_t physical_freestart; 294 extern paddr_t physical_end; 295 extern paddr_t physical_freeend; 296 extern unsigned int free_pages; 297 extern int max_processes; 298 299 vaddr_t virtual_avail; 300 vaddr_t virtual_end; 301 vaddr_t pmap_curmaxkvaddr; 302 303 vaddr_t avail_start; 304 vaddr_t avail_end; 305 306 extern pv_addr_t systempage; 307 308 /* Variables used by the L1 page table queue code */ 309 SIMPLEQ_HEAD(l1pt_queue, l1pt); 310 static struct l1pt_queue l1pt_static_queue; /* head of our static l1 queue */ 311 static int l1pt_static_queue_count; /* items in the static l1 queue */ 312 static int l1pt_static_create_count; /* static l1 items created */ 313 static struct l1pt_queue l1pt_queue; /* head of our l1 queue */ 314 static int l1pt_queue_count; /* items in the l1 queue */ 315 static int l1pt_create_count; /* stat - L1's create count */ 316 static int l1pt_reuse_count; /* stat - L1's reused count */ 317 318 /* Local function prototypes (not used outside this file) */ 319 void pmap_pinit __P((struct pmap *)); 320 void pmap_freepagedir __P((struct pmap *)); 321 322 /* Other function prototypes */ 323 extern void bzero_page __P((vaddr_t)); 324 extern void bcopy_page __P((vaddr_t, vaddr_t)); 325 326 struct l1pt *pmap_alloc_l1pt __P((void)); 327 static __inline void pmap_map_in_l1 __P((struct pmap *pmap, vaddr_t va, 328 vaddr_t l2pa, boolean_t)); 329 330 static pt_entry_t *pmap_map_ptes __P((struct pmap *)); 331 static void pmap_unmap_ptes __P((struct pmap *)); 332 333 __inline static void pmap_vac_me_harder __P((struct pmap *, struct vm_page *, 334 pt_entry_t *, boolean_t)); 335 static void pmap_vac_me_kpmap __P((struct pmap *, struct vm_page *, 336 pt_entry_t *, boolean_t)); 337 static void pmap_vac_me_user __P((struct pmap *, struct vm_page *, 338 pt_entry_t *, boolean_t)); 339 340 /* 341 * real definition of pv_entry. 342 */ 343 344 struct pv_entry { 345 struct pv_entry *pv_next; /* next pv_entry */ 346 struct pmap *pv_pmap; /* pmap where mapping lies */ 347 vaddr_t pv_va; /* virtual address for mapping */ 348 int pv_flags; /* flags */ 349 struct vm_page *pv_ptp; /* vm_page for the ptp */ 350 }; 351 352 /* 353 * pv_entrys are dynamically allocated in chunks from a single page. 354 * we keep track of how many pv_entrys are in use for each page and 355 * we can free pv_entry pages if needed. there is one lock for the 356 * entire allocation system. 357 */ 358 359 struct pv_page_info { 360 TAILQ_ENTRY(pv_page) pvpi_list; 361 struct pv_entry *pvpi_pvfree; 362 int pvpi_nfree; 363 }; 364 365 /* 366 * number of pv_entry's in a pv_page 367 * (note: won't work on systems where NPBG isn't a constant) 368 */ 369 370 #define PVE_PER_PVPAGE ((NBPG - sizeof(struct pv_page_info)) / \ 371 sizeof(struct pv_entry)) 372 373 /* 374 * a pv_page: where pv_entrys are allocated from 375 */ 376 377 struct pv_page { 378 struct pv_page_info pvinfo; 379 struct pv_entry pvents[PVE_PER_PVPAGE]; 380 }; 381 382 #ifdef MYCROFT_HACK 383 int mycroft_hack = 0; 384 #endif 385 386 /* Function to set the debug level of the pmap code */ 387 388 #ifdef PMAP_DEBUG 389 void 390 pmap_debug(int level) 391 { 392 pmap_debug_level = level; 393 printf("pmap_debug: level=%d\n", pmap_debug_level); 394 } 395 #endif /* PMAP_DEBUG */ 396 397 __inline static boolean_t 398 pmap_is_curpmap(struct pmap *pmap) 399 { 400 401 if ((curproc && curproc->p_vmspace->vm_map.pmap == pmap) || 402 pmap == pmap_kernel()) 403 return (TRUE); 404 405 return (FALSE); 406 } 407 408 #include "isadma.h" 409 410 #if NISADMA > 0 411 /* 412 * Used to protect memory for ISA DMA bounce buffers. If, when loading 413 * pages into the system, memory intersects with any of these ranges, 414 * the intersecting memory will be loaded into a lower-priority free list. 415 */ 416 bus_dma_segment_t *pmap_isa_dma_ranges; 417 int pmap_isa_dma_nranges; 418 419 /* 420 * Check if a memory range intersects with an ISA DMA range, and 421 * return the page-rounded intersection if it does. The intersection 422 * will be placed on a lower-priority free list. 423 */ 424 static boolean_t 425 pmap_isa_dma_range_intersect(paddr_t pa, psize_t size, paddr_t *pap, 426 psize_t *sizep) 427 { 428 bus_dma_segment_t *ds; 429 int i; 430 431 if (pmap_isa_dma_ranges == NULL) 432 return (FALSE); 433 434 for (i = 0, ds = pmap_isa_dma_ranges; 435 i < pmap_isa_dma_nranges; i++, ds++) { 436 if (ds->ds_addr <= pa && pa < (ds->ds_addr + ds->ds_len)) { 437 /* 438 * Beginning of region intersects with this range. 439 */ 440 *pap = trunc_page(pa); 441 *sizep = round_page(min(pa + size, 442 ds->ds_addr + ds->ds_len) - pa); 443 return (TRUE); 444 } 445 if (pa < ds->ds_addr && ds->ds_addr < (pa + size)) { 446 /* 447 * End of region intersects with this range. 448 */ 449 *pap = trunc_page(ds->ds_addr); 450 *sizep = round_page(min((pa + size) - ds->ds_addr, 451 ds->ds_len)); 452 return (TRUE); 453 } 454 } 455 456 /* 457 * No intersection found. 458 */ 459 return (FALSE); 460 } 461 #endif /* NISADMA > 0 */ 462 463 /* 464 * p v _ e n t r y f u n c t i o n s 465 */ 466 467 /* 468 * pv_entry allocation functions: 469 * the main pv_entry allocation functions are: 470 * pmap_alloc_pv: allocate a pv_entry structure 471 * pmap_free_pv: free one pv_entry 472 * pmap_free_pvs: free a list of pv_entrys 473 * 474 * the rest are helper functions 475 */ 476 477 /* 478 * pmap_alloc_pv: inline function to allocate a pv_entry structure 479 * => we lock pvalloc_lock 480 * => if we fail, we call out to pmap_alloc_pvpage 481 * => 3 modes: 482 * ALLOCPV_NEED = we really need a pv_entry, even if we have to steal it 483 * ALLOCPV_TRY = we want a pv_entry, but not enough to steal 484 * ALLOCPV_NONEED = we are trying to grow our free list, don't really need 485 * one now 486 * 487 * "try" is for optional functions like pmap_copy(). 488 */ 489 490 __inline static struct pv_entry * 491 pmap_alloc_pv(struct pmap *pmap, int mode) 492 { 493 struct pv_page *pvpage; 494 struct pv_entry *pv; 495 496 simple_lock(&pvalloc_lock); 497 498 pvpage = TAILQ_FIRST(&pv_freepages); 499 500 if (pvpage != NULL) { 501 pvpage->pvinfo.pvpi_nfree--; 502 if (pvpage->pvinfo.pvpi_nfree == 0) { 503 /* nothing left in this one? */ 504 TAILQ_REMOVE(&pv_freepages, pvpage, pvinfo.pvpi_list); 505 } 506 pv = pvpage->pvinfo.pvpi_pvfree; 507 KASSERT(pv); 508 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 509 pv_nfpvents--; /* took one from pool */ 510 } else { 511 pv = NULL; /* need more of them */ 512 } 513 514 /* 515 * if below low water mark or we didn't get a pv_entry we try and 516 * create more pv_entrys ... 517 */ 518 519 if (pv_nfpvents < PVE_LOWAT || pv == NULL) { 520 if (pv == NULL) 521 pv = pmap_alloc_pvpage(pmap, (mode == ALLOCPV_TRY) ? 522 mode : ALLOCPV_NEED); 523 else 524 (void) pmap_alloc_pvpage(pmap, ALLOCPV_NONEED); 525 } 526 527 simple_unlock(&pvalloc_lock); 528 return(pv); 529 } 530 531 /* 532 * pmap_alloc_pvpage: maybe allocate a new pvpage 533 * 534 * if need_entry is false: try and allocate a new pv_page 535 * if need_entry is true: try and allocate a new pv_page and return a 536 * new pv_entry from it. if we are unable to allocate a pv_page 537 * we make a last ditch effort to steal a pv_page from some other 538 * mapping. if that fails, we panic... 539 * 540 * => we assume that the caller holds pvalloc_lock 541 */ 542 543 static struct pv_entry * 544 pmap_alloc_pvpage(struct pmap *pmap, int mode) 545 { 546 struct vm_page *pg; 547 struct pv_page *pvpage; 548 struct pv_entry *pv; 549 int s; 550 551 /* 552 * if we need_entry and we've got unused pv_pages, allocate from there 553 */ 554 555 pvpage = TAILQ_FIRST(&pv_unusedpgs); 556 if (mode != ALLOCPV_NONEED && pvpage != NULL) { 557 558 /* move it to pv_freepages list */ 559 TAILQ_REMOVE(&pv_unusedpgs, pvpage, pvinfo.pvpi_list); 560 TAILQ_INSERT_HEAD(&pv_freepages, pvpage, pvinfo.pvpi_list); 561 562 /* allocate a pv_entry */ 563 pvpage->pvinfo.pvpi_nfree--; /* can't go to zero */ 564 pv = pvpage->pvinfo.pvpi_pvfree; 565 KASSERT(pv); 566 pvpage->pvinfo.pvpi_pvfree = pv->pv_next; 567 568 pv_nfpvents--; /* took one from pool */ 569 return(pv); 570 } 571 572 /* 573 * see if we've got a cached unmapped VA that we can map a page in. 574 * if not, try to allocate one. 575 */ 576 577 578 if (pv_cachedva == 0) { 579 s = splvm(); 580 pv_cachedva = uvm_km_kmemalloc(kmem_map, NULL, 581 PAGE_SIZE, UVM_KMF_TRYLOCK|UVM_KMF_VALLOC); 582 splx(s); 583 if (pv_cachedva == 0) { 584 return (NULL); 585 } 586 } 587 588 pg = uvm_pagealloc(NULL, pv_cachedva - vm_map_min(kernel_map), NULL, 589 UVM_PGA_USERESERVE); 590 591 if (pg == NULL) 592 return (NULL); 593 pg->flags &= ~PG_BUSY; /* never busy */ 594 595 /* 596 * add a mapping for our new pv_page and free its entrys (save one!) 597 * 598 * NOTE: If we are allocating a PV page for the kernel pmap, the 599 * pmap is already locked! (...but entering the mapping is safe...) 600 */ 601 602 pmap_kenter_pa(pv_cachedva, VM_PAGE_TO_PHYS(pg), 603 VM_PROT_READ|VM_PROT_WRITE); 604 pmap_update(pmap_kernel()); 605 pvpage = (struct pv_page *) pv_cachedva; 606 pv_cachedva = 0; 607 return (pmap_add_pvpage(pvpage, mode != ALLOCPV_NONEED)); 608 } 609 610 /* 611 * pmap_add_pvpage: add a pv_page's pv_entrys to the free list 612 * 613 * => caller must hold pvalloc_lock 614 * => if need_entry is true, we allocate and return one pv_entry 615 */ 616 617 static struct pv_entry * 618 pmap_add_pvpage(struct pv_page *pvp, boolean_t need_entry) 619 { 620 int tofree, lcv; 621 622 /* do we need to return one? */ 623 tofree = (need_entry) ? PVE_PER_PVPAGE - 1 : PVE_PER_PVPAGE; 624 625 pvp->pvinfo.pvpi_pvfree = NULL; 626 pvp->pvinfo.pvpi_nfree = tofree; 627 for (lcv = 0 ; lcv < tofree ; lcv++) { 628 pvp->pvents[lcv].pv_next = pvp->pvinfo.pvpi_pvfree; 629 pvp->pvinfo.pvpi_pvfree = &pvp->pvents[lcv]; 630 } 631 if (need_entry) 632 TAILQ_INSERT_TAIL(&pv_freepages, pvp, pvinfo.pvpi_list); 633 else 634 TAILQ_INSERT_TAIL(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 635 pv_nfpvents += tofree; 636 return((need_entry) ? &pvp->pvents[lcv] : NULL); 637 } 638 639 /* 640 * pmap_free_pv_doit: actually free a pv_entry 641 * 642 * => do not call this directly! instead use either 643 * 1. pmap_free_pv ==> free a single pv_entry 644 * 2. pmap_free_pvs => free a list of pv_entrys 645 * => we must be holding pvalloc_lock 646 */ 647 648 __inline static void 649 pmap_free_pv_doit(struct pv_entry *pv) 650 { 651 struct pv_page *pvp; 652 653 pvp = (struct pv_page *) arm_trunc_page((vaddr_t)pv); 654 pv_nfpvents++; 655 pvp->pvinfo.pvpi_nfree++; 656 657 /* nfree == 1 => fully allocated page just became partly allocated */ 658 if (pvp->pvinfo.pvpi_nfree == 1) { 659 TAILQ_INSERT_HEAD(&pv_freepages, pvp, pvinfo.pvpi_list); 660 } 661 662 /* free it */ 663 pv->pv_next = pvp->pvinfo.pvpi_pvfree; 664 pvp->pvinfo.pvpi_pvfree = pv; 665 666 /* 667 * are all pv_page's pv_entry's free? move it to unused queue. 668 */ 669 670 if (pvp->pvinfo.pvpi_nfree == PVE_PER_PVPAGE) { 671 TAILQ_REMOVE(&pv_freepages, pvp, pvinfo.pvpi_list); 672 TAILQ_INSERT_HEAD(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 673 } 674 } 675 676 /* 677 * pmap_free_pv: free a single pv_entry 678 * 679 * => we gain the pvalloc_lock 680 */ 681 682 __inline static void 683 pmap_free_pv(struct pmap *pmap, struct pv_entry *pv) 684 { 685 simple_lock(&pvalloc_lock); 686 pmap_free_pv_doit(pv); 687 688 /* 689 * Can't free the PV page if the PV entries were associated with 690 * the kernel pmap; the pmap is already locked. 691 */ 692 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 693 pmap != pmap_kernel()) 694 pmap_free_pvpage(); 695 696 simple_unlock(&pvalloc_lock); 697 } 698 699 /* 700 * pmap_free_pvs: free a list of pv_entrys 701 * 702 * => we gain the pvalloc_lock 703 */ 704 705 __inline static void 706 pmap_free_pvs(struct pmap *pmap, struct pv_entry *pvs) 707 { 708 struct pv_entry *nextpv; 709 710 simple_lock(&pvalloc_lock); 711 712 for ( /* null */ ; pvs != NULL ; pvs = nextpv) { 713 nextpv = pvs->pv_next; 714 pmap_free_pv_doit(pvs); 715 } 716 717 /* 718 * Can't free the PV page if the PV entries were associated with 719 * the kernel pmap; the pmap is already locked. 720 */ 721 if (pv_nfpvents > PVE_HIWAT && TAILQ_FIRST(&pv_unusedpgs) != NULL && 722 pmap != pmap_kernel()) 723 pmap_free_pvpage(); 724 725 simple_unlock(&pvalloc_lock); 726 } 727 728 729 /* 730 * pmap_free_pvpage: try and free an unused pv_page structure 731 * 732 * => assume caller is holding the pvalloc_lock and that 733 * there is a page on the pv_unusedpgs list 734 * => if we can't get a lock on the kmem_map we try again later 735 */ 736 737 static void 738 pmap_free_pvpage(void) 739 { 740 int s; 741 struct vm_map *map; 742 struct vm_map_entry *dead_entries; 743 struct pv_page *pvp; 744 745 s = splvm(); /* protect kmem_map */ 746 747 pvp = TAILQ_FIRST(&pv_unusedpgs); 748 749 /* 750 * note: watch out for pv_initpage which is allocated out of 751 * kernel_map rather than kmem_map. 752 */ 753 if (pvp == pv_initpage) 754 map = kernel_map; 755 else 756 map = kmem_map; 757 if (vm_map_lock_try(map)) { 758 759 /* remove pvp from pv_unusedpgs */ 760 TAILQ_REMOVE(&pv_unusedpgs, pvp, pvinfo.pvpi_list); 761 762 /* unmap the page */ 763 dead_entries = NULL; 764 uvm_unmap_remove(map, (vaddr_t)pvp, ((vaddr_t)pvp) + PAGE_SIZE, 765 &dead_entries); 766 vm_map_unlock(map); 767 768 if (dead_entries != NULL) 769 uvm_unmap_detach(dead_entries, 0); 770 771 pv_nfpvents -= PVE_PER_PVPAGE; /* update free count */ 772 } 773 if (pvp == pv_initpage) 774 /* no more initpage, we've freed it */ 775 pv_initpage = NULL; 776 777 splx(s); 778 } 779 780 /* 781 * main pv_entry manipulation functions: 782 * pmap_enter_pv: enter a mapping onto a vm_page list 783 * pmap_remove_pv: remove a mappiing from a vm_page list 784 * 785 * NOTE: pmap_enter_pv expects to lock the pvh itself 786 * pmap_remove_pv expects te caller to lock the pvh before calling 787 */ 788 789 /* 790 * pmap_enter_pv: enter a mapping onto a vm_page lst 791 * 792 * => caller should hold the proper lock on pmap_main_lock 793 * => caller should have pmap locked 794 * => we will gain the lock on the vm_page and allocate the new pv_entry 795 * => caller should adjust ptp's wire_count before calling 796 * => caller should not adjust pmap's wire_count 797 */ 798 799 __inline static void 800 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap, 801 vaddr_t va, struct vm_page *ptp, int flags) 802 { 803 pve->pv_pmap = pmap; 804 pve->pv_va = va; 805 pve->pv_ptp = ptp; /* NULL for kernel pmap */ 806 pve->pv_flags = flags; 807 simple_lock(&pg->mdpage.pvh_slock); /* lock vm_page */ 808 pve->pv_next = pg->mdpage.pvh_list; /* add to ... */ 809 pg->mdpage.pvh_list = pve; /* ... locked list */ 810 simple_unlock(&pg->mdpage.pvh_slock); /* unlock, done! */ 811 if (pve->pv_flags & PVF_WIRED) 812 ++pmap->pm_stats.wired_count; 813 } 814 815 /* 816 * pmap_remove_pv: try to remove a mapping from a pv_list 817 * 818 * => caller should hold proper lock on pmap_main_lock 819 * => pmap should be locked 820 * => caller should hold lock on vm_page [so that attrs can be adjusted] 821 * => caller should adjust ptp's wire_count and free PTP if needed 822 * => caller should NOT adjust pmap's wire_count 823 * => we return the removed pve 824 */ 825 826 __inline static struct pv_entry * 827 pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va) 828 { 829 struct pv_entry *pve, **prevptr; 830 831 prevptr = &pg->mdpage.pvh_list; /* previous pv_entry pointer */ 832 pve = *prevptr; 833 while (pve) { 834 if (pve->pv_pmap == pmap && pve->pv_va == va) { /* match? */ 835 *prevptr = pve->pv_next; /* remove it! */ 836 if (pve->pv_flags & PVF_WIRED) 837 --pmap->pm_stats.wired_count; 838 break; 839 } 840 prevptr = &pve->pv_next; /* previous pointer */ 841 pve = pve->pv_next; /* advance */ 842 } 843 return(pve); /* return removed pve */ 844 } 845 846 /* 847 * 848 * pmap_modify_pv: Update pv flags 849 * 850 * => caller should hold lock on vm_page [so that attrs can be adjusted] 851 * => caller should NOT adjust pmap's wire_count 852 * => caller must call pmap_vac_me_harder() if writable status of a page 853 * may have changed. 854 * => we return the old flags 855 * 856 * Modify a physical-virtual mapping in the pv table 857 */ 858 859 static /* __inline */ u_int 860 pmap_modify_pv(struct pmap *pmap, vaddr_t va, struct vm_page *pg, 861 u_int bic_mask, u_int eor_mask) 862 { 863 struct pv_entry *npv; 864 u_int flags, oflags; 865 866 /* 867 * There is at least one VA mapping this page. 868 */ 869 870 for (npv = pg->mdpage.pvh_list; npv; npv = npv->pv_next) { 871 if (pmap == npv->pv_pmap && va == npv->pv_va) { 872 oflags = npv->pv_flags; 873 npv->pv_flags = flags = 874 ((oflags & ~bic_mask) ^ eor_mask); 875 if ((flags ^ oflags) & PVF_WIRED) { 876 if (flags & PVF_WIRED) 877 ++pmap->pm_stats.wired_count; 878 else 879 --pmap->pm_stats.wired_count; 880 } 881 return (oflags); 882 } 883 } 884 return (0); 885 } 886 887 /* 888 * Map the specified level 2 pagetable into the level 1 page table for 889 * the given pmap to cover a chunk of virtual address space starting from the 890 * address specified. 891 */ 892 static __inline void 893 pmap_map_in_l1(struct pmap *pmap, vaddr_t va, paddr_t l2pa, boolean_t selfref) 894 { 895 vaddr_t ptva; 896 897 /* Calculate the index into the L1 page table. */ 898 ptva = (va >> L1_S_SHIFT) & ~3; 899 900 /* Map page table into the L1. */ 901 pmap->pm_pdir[ptva + 0] = L1_C_PROTO | (l2pa + 0x000); 902 pmap->pm_pdir[ptva + 1] = L1_C_PROTO | (l2pa + 0x400); 903 pmap->pm_pdir[ptva + 2] = L1_C_PROTO | (l2pa + 0x800); 904 pmap->pm_pdir[ptva + 3] = L1_C_PROTO | (l2pa + 0xc00); 905 906 /* Map the page table into the page table area. */ 907 if (selfref) 908 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = L2_S_PROTO | l2pa | 909 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE); 910 } 911 912 #if 0 913 static __inline void 914 pmap_unmap_in_l1(struct pmap *pmap, vaddr_t va) 915 { 916 vaddr_t ptva; 917 918 /* Calculate the index into the L1 page table. */ 919 ptva = (va >> L1_S_SHIFT) & ~3; 920 921 /* Unmap page table from the L1. */ 922 pmap->pm_pdir[ptva + 0] = 0; 923 pmap->pm_pdir[ptva + 1] = 0; 924 pmap->pm_pdir[ptva + 2] = 0; 925 pmap->pm_pdir[ptva + 3] = 0; 926 927 /* Unmap the page table from the page table area. */ 928 *((pt_entry_t *)(pmap->pm_vptpt + ptva)) = 0; 929 } 930 #endif 931 932 /* 933 * Used to map a range of physical addresses into kernel 934 * virtual address space. 935 * 936 * For now, VM is already on, we only need to map the 937 * specified memory. 938 */ 939 vaddr_t 940 pmap_map(vaddr_t va, paddr_t spa, paddr_t epa, vm_prot_t prot) 941 { 942 while (spa < epa) { 943 pmap_kenter_pa(va, spa, prot); 944 va += NBPG; 945 spa += NBPG; 946 } 947 pmap_update(pmap_kernel()); 948 return(va); 949 } 950 951 952 /* 953 * void pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 954 * 955 * bootstrap the pmap system. This is called from initarm and allows 956 * the pmap system to initailise any structures it requires. 957 * 958 * Currently this sets up the kernel_pmap that is statically allocated 959 * and also allocated virtual addresses for certain page hooks. 960 * Currently the only one page hook is allocated that is used 961 * to zero physical pages of memory. 962 * It also initialises the start and end address of the kernel data space. 963 */ 964 extern paddr_t physical_freestart; 965 extern paddr_t physical_freeend; 966 967 char *boot_head; 968 969 void 970 pmap_bootstrap(pd_entry_t *kernel_l1pt, pv_addr_t kernel_ptpt) 971 { 972 pt_entry_t *pte; 973 int loop; 974 paddr_t start, end; 975 #if NISADMA > 0 976 paddr_t istart; 977 psize_t isize; 978 #endif 979 980 pmap_kernel()->pm_pdir = kernel_l1pt; 981 pmap_kernel()->pm_pptpt = kernel_ptpt.pv_pa; 982 pmap_kernel()->pm_vptpt = kernel_ptpt.pv_va; 983 simple_lock_init(&pmap_kernel()->pm_lock); 984 pmap_kernel()->pm_obj.pgops = NULL; 985 TAILQ_INIT(&(pmap_kernel()->pm_obj.memq)); 986 pmap_kernel()->pm_obj.uo_npages = 0; 987 pmap_kernel()->pm_obj.uo_refs = 1; 988 989 /* 990 * Initialize PAGE_SIZE-dependent variables. 991 */ 992 uvm_setpagesize(); 993 994 loop = 0; 995 while (loop < bootconfig.dramblocks) { 996 start = (paddr_t)bootconfig.dram[loop].address; 997 end = start + (bootconfig.dram[loop].pages * NBPG); 998 if (start < physical_freestart) 999 start = physical_freestart; 1000 if (end > physical_freeend) 1001 end = physical_freeend; 1002 #if 0 1003 printf("%d: %lx -> %lx\n", loop, start, end - 1); 1004 #endif 1005 #if NISADMA > 0 1006 if (pmap_isa_dma_range_intersect(start, end - start, 1007 &istart, &isize)) { 1008 /* 1009 * Place the pages that intersect with the 1010 * ISA DMA range onto the ISA DMA free list. 1011 */ 1012 #if 0 1013 printf(" ISADMA 0x%lx -> 0x%lx\n", istart, 1014 istart + isize - 1); 1015 #endif 1016 uvm_page_physload(atop(istart), 1017 atop(istart + isize), atop(istart), 1018 atop(istart + isize), VM_FREELIST_ISADMA); 1019 1020 /* 1021 * Load the pieces that come before 1022 * the intersection into the default 1023 * free list. 1024 */ 1025 if (start < istart) { 1026 #if 0 1027 printf(" BEFORE 0x%lx -> 0x%lx\n", 1028 start, istart - 1); 1029 #endif 1030 uvm_page_physload(atop(start), 1031 atop(istart), atop(start), 1032 atop(istart), VM_FREELIST_DEFAULT); 1033 } 1034 1035 /* 1036 * Load the pieces that come after 1037 * the intersection into the default 1038 * free list. 1039 */ 1040 if ((istart + isize) < end) { 1041 #if 0 1042 printf(" AFTER 0x%lx -> 0x%lx\n", 1043 (istart + isize), end - 1); 1044 #endif 1045 uvm_page_physload(atop(istart + isize), 1046 atop(end), atop(istart + isize), 1047 atop(end), VM_FREELIST_DEFAULT); 1048 } 1049 } else { 1050 uvm_page_physload(atop(start), atop(end), 1051 atop(start), atop(end), VM_FREELIST_DEFAULT); 1052 } 1053 #else /* NISADMA > 0 */ 1054 uvm_page_physload(atop(start), atop(end), 1055 atop(start), atop(end), VM_FREELIST_DEFAULT); 1056 #endif /* NISADMA > 0 */ 1057 ++loop; 1058 } 1059 1060 virtual_avail = KERNEL_VM_BASE; 1061 virtual_end = KERNEL_VM_BASE + KERNEL_VM_SIZE; 1062 1063 /* 1064 * now we allocate the "special" VAs which are used for tmp mappings 1065 * by the pmap (and other modules). we allocate the VAs by advancing 1066 * virtual_avail (note that there are no pages mapped at these VAs). 1067 * we find the PTE that maps the allocated VA via the linear PTE 1068 * mapping. 1069 */ 1070 1071 pte = ((pt_entry_t *) PTE_BASE) + atop(virtual_avail); 1072 1073 csrcp = virtual_avail; csrc_pte = pte; 1074 virtual_avail += PAGE_SIZE; pte++; 1075 1076 cdstp = virtual_avail; cdst_pte = pte; 1077 virtual_avail += PAGE_SIZE; pte++; 1078 1079 memhook = (char *) virtual_avail; /* don't need pte */ 1080 virtual_avail += PAGE_SIZE; pte++; 1081 1082 msgbufaddr = (caddr_t) virtual_avail; /* don't need pte */ 1083 virtual_avail += round_page(MSGBUFSIZE); 1084 pte += atop(round_page(MSGBUFSIZE)); 1085 1086 /* 1087 * init the static-global locks and global lists. 1088 */ 1089 spinlockinit(&pmap_main_lock, "pmaplk", 0); 1090 simple_lock_init(&pvalloc_lock); 1091 simple_lock_init(&pmaps_lock); 1092 LIST_INIT(&pmaps); 1093 TAILQ_INIT(&pv_freepages); 1094 TAILQ_INIT(&pv_unusedpgs); 1095 1096 /* 1097 * initialize the pmap pool. 1098 */ 1099 1100 pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1101 &pool_allocator_nointr); 1102 1103 cpu_dcache_wbinv_all(); 1104 } 1105 1106 /* 1107 * void pmap_init(void) 1108 * 1109 * Initialize the pmap module. 1110 * Called by vm_init() in vm/vm_init.c in order to initialise 1111 * any structures that the pmap system needs to map virtual memory. 1112 */ 1113 1114 extern int physmem; 1115 1116 void 1117 pmap_init(void) 1118 { 1119 1120 /* 1121 * Set the available memory vars - These do not map to real memory 1122 * addresses and cannot as the physical memory is fragmented. 1123 * They are used by ps for %mem calculations. 1124 * One could argue whether this should be the entire memory or just 1125 * the memory that is useable in a user process. 1126 */ 1127 avail_start = 0; 1128 avail_end = physmem * NBPG; 1129 1130 /* 1131 * now we need to free enough pv_entry structures to allow us to get 1132 * the kmem_map/kmem_object allocated and inited (done after this 1133 * function is finished). to do this we allocate one bootstrap page out 1134 * of kernel_map and use it to provide an initial pool of pv_entry 1135 * structures. we never free this page. 1136 */ 1137 1138 pv_initpage = (struct pv_page *) uvm_km_alloc(kernel_map, PAGE_SIZE); 1139 if (pv_initpage == NULL) 1140 panic("pmap_init: pv_initpage"); 1141 pv_cachedva = 0; /* a VA we have allocated but not used yet */ 1142 pv_nfpvents = 0; 1143 (void) pmap_add_pvpage(pv_initpage, FALSE); 1144 1145 pmap_initialized = TRUE; 1146 1147 /* Initialise our L1 page table queues and counters */ 1148 SIMPLEQ_INIT(&l1pt_static_queue); 1149 l1pt_static_queue_count = 0; 1150 l1pt_static_create_count = 0; 1151 SIMPLEQ_INIT(&l1pt_queue); 1152 l1pt_queue_count = 0; 1153 l1pt_create_count = 0; 1154 l1pt_reuse_count = 0; 1155 } 1156 1157 /* 1158 * pmap_postinit() 1159 * 1160 * This routine is called after the vm and kmem subsystems have been 1161 * initialised. This allows the pmap code to perform any initialisation 1162 * that can only be done one the memory allocation is in place. 1163 */ 1164 1165 void 1166 pmap_postinit(void) 1167 { 1168 int loop; 1169 struct l1pt *pt; 1170 1171 #ifdef PMAP_STATIC_L1S 1172 for (loop = 0; loop < PMAP_STATIC_L1S; ++loop) { 1173 #else /* PMAP_STATIC_L1S */ 1174 for (loop = 0; loop < max_processes; ++loop) { 1175 #endif /* PMAP_STATIC_L1S */ 1176 /* Allocate a L1 page table */ 1177 pt = pmap_alloc_l1pt(); 1178 if (!pt) 1179 panic("Cannot allocate static L1 page tables\n"); 1180 1181 /* Clean it */ 1182 bzero((void *)pt->pt_va, L1_TABLE_SIZE); 1183 pt->pt_flags |= (PTFLAG_STATIC | PTFLAG_CLEAN); 1184 /* Add the page table to the queue */ 1185 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pt, pt_queue); 1186 ++l1pt_static_queue_count; 1187 ++l1pt_static_create_count; 1188 } 1189 } 1190 1191 1192 /* 1193 * Create and return a physical map. 1194 * 1195 * If the size specified for the map is zero, the map is an actual physical 1196 * map, and may be referenced by the hardware. 1197 * 1198 * If the size specified is non-zero, the map will be used in software only, 1199 * and is bounded by that size. 1200 */ 1201 1202 pmap_t 1203 pmap_create(void) 1204 { 1205 struct pmap *pmap; 1206 1207 /* 1208 * Fetch pmap entry from the pool 1209 */ 1210 1211 pmap = pool_get(&pmap_pmap_pool, PR_WAITOK); 1212 /* XXX is this really needed! */ 1213 memset(pmap, 0, sizeof(*pmap)); 1214 1215 simple_lock_init(&pmap->pm_obj.vmobjlock); 1216 pmap->pm_obj.pgops = NULL; /* currently not a mappable object */ 1217 TAILQ_INIT(&pmap->pm_obj.memq); 1218 pmap->pm_obj.uo_npages = 0; 1219 pmap->pm_obj.uo_refs = 1; 1220 pmap->pm_stats.wired_count = 0; 1221 pmap->pm_stats.resident_count = 1; 1222 pmap->pm_ptphint = NULL; 1223 1224 /* Now init the machine part of the pmap */ 1225 pmap_pinit(pmap); 1226 return(pmap); 1227 } 1228 1229 /* 1230 * pmap_alloc_l1pt() 1231 * 1232 * This routine allocates physical and virtual memory for a L1 page table 1233 * and wires it. 1234 * A l1pt structure is returned to describe the allocated page table. 1235 * 1236 * This routine is allowed to fail if the required memory cannot be allocated. 1237 * In this case NULL is returned. 1238 */ 1239 1240 struct l1pt * 1241 pmap_alloc_l1pt(void) 1242 { 1243 paddr_t pa; 1244 vaddr_t va; 1245 struct l1pt *pt; 1246 int error; 1247 struct vm_page *m; 1248 pt_entry_t *ptes; 1249 1250 /* Allocate virtual address space for the L1 page table */ 1251 va = uvm_km_valloc(kernel_map, L1_TABLE_SIZE); 1252 if (va == 0) { 1253 #ifdef DIAGNOSTIC 1254 PDEBUG(0, 1255 printf("pmap: Cannot allocate pageable memory for L1\n")); 1256 #endif /* DIAGNOSTIC */ 1257 return(NULL); 1258 } 1259 1260 /* Allocate memory for the l1pt structure */ 1261 pt = (struct l1pt *)malloc(sizeof(struct l1pt), M_VMPMAP, M_WAITOK); 1262 1263 /* 1264 * Allocate pages from the VM system. 1265 */ 1266 TAILQ_INIT(&pt->pt_plist); 1267 error = uvm_pglistalloc(L1_TABLE_SIZE, physical_start, physical_end, 1268 L1_TABLE_SIZE, 0, &pt->pt_plist, 1, M_WAITOK); 1269 if (error) { 1270 #ifdef DIAGNOSTIC 1271 PDEBUG(0, 1272 printf("pmap: Cannot allocate physical mem for L1 (%d)\n", 1273 error)); 1274 #endif /* DIAGNOSTIC */ 1275 /* Release the resources we already have claimed */ 1276 free(pt, M_VMPMAP); 1277 uvm_km_free(kernel_map, va, L1_TABLE_SIZE); 1278 return(NULL); 1279 } 1280 1281 /* Map our physical pages into our virtual space */ 1282 pt->pt_va = va; 1283 m = TAILQ_FIRST(&pt->pt_plist); 1284 ptes = pmap_map_ptes(pmap_kernel()); 1285 while (m && va < (pt->pt_va + L1_TABLE_SIZE)) { 1286 pa = VM_PAGE_TO_PHYS(m); 1287 1288 pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE); 1289 1290 /* Revoke cacheability and bufferability */ 1291 /* XXX should be done better than this */ 1292 ptes[arm_btop(va)] &= ~(L2_C | L2_B); 1293 1294 va += NBPG; 1295 m = m->pageq.tqe_next; 1296 } 1297 pmap_unmap_ptes(pmap_kernel()); 1298 pmap_update(pmap_kernel()); 1299 1300 #ifdef DIAGNOSTIC 1301 if (m) 1302 panic("pmap_alloc_l1pt: pglist not empty\n"); 1303 #endif /* DIAGNOSTIC */ 1304 1305 pt->pt_flags = 0; 1306 return(pt); 1307 } 1308 1309 /* 1310 * Free a L1 page table previously allocated with pmap_alloc_l1pt(). 1311 */ 1312 static void 1313 pmap_free_l1pt(struct l1pt *pt) 1314 { 1315 /* Separate the physical memory for the virtual space */ 1316 pmap_kremove(pt->pt_va, L1_TABLE_SIZE); 1317 pmap_update(pmap_kernel()); 1318 1319 /* Return the physical memory */ 1320 uvm_pglistfree(&pt->pt_plist); 1321 1322 /* Free the virtual space */ 1323 uvm_km_free(kernel_map, pt->pt_va, L1_TABLE_SIZE); 1324 1325 /* Free the l1pt structure */ 1326 free(pt, M_VMPMAP); 1327 } 1328 1329 /* 1330 * Allocate a page directory. 1331 * This routine will either allocate a new page directory from the pool 1332 * of L1 page tables currently held by the kernel or it will allocate 1333 * a new one via pmap_alloc_l1pt(). 1334 * It will then initialise the l1 page table for use. 1335 * 1336 * XXX must tidy up and fix this code, not happy about how it does the pmaps_locking 1337 */ 1338 static int 1339 pmap_allocpagedir(struct pmap *pmap) 1340 { 1341 paddr_t pa; 1342 struct l1pt *pt; 1343 pt_entry_t *pte; 1344 1345 PDEBUG(0, printf("pmap_allocpagedir(%p)\n", pmap)); 1346 1347 /* Do we have any spare L1's lying around ? */ 1348 if (l1pt_static_queue_count) { 1349 --l1pt_static_queue_count; 1350 pt = l1pt_static_queue.sqh_first; 1351 SIMPLEQ_REMOVE_HEAD(&l1pt_static_queue, pt, pt_queue); 1352 } else if (l1pt_queue_count) { 1353 --l1pt_queue_count; 1354 pt = l1pt_queue.sqh_first; 1355 SIMPLEQ_REMOVE_HEAD(&l1pt_queue, pt, pt_queue); 1356 ++l1pt_reuse_count; 1357 } else { 1358 pt = pmap_alloc_l1pt(); 1359 if (!pt) 1360 return(ENOMEM); 1361 ++l1pt_create_count; 1362 } 1363 1364 /* Store the pointer to the l1 descriptor in the pmap. */ 1365 pmap->pm_l1pt = pt; 1366 1367 /* Get the physical address of the start of the l1 */ 1368 pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pt->pt_plist)); 1369 1370 /* Store the virtual address of the l1 in the pmap. */ 1371 pmap->pm_pdir = (pd_entry_t *)pt->pt_va; 1372 1373 /* Clean the L1 if it is dirty */ 1374 if (!(pt->pt_flags & PTFLAG_CLEAN)) 1375 bzero((void *)pmap->pm_pdir, (L1_TABLE_SIZE - KERNEL_PD_SIZE)); 1376 1377 /* Allocate a page table to map all the page tables for this pmap */ 1378 1379 #ifdef DIAGNOSTIC 1380 if (pmap->pm_vptpt) { 1381 /* XXX What if we have one already ? */ 1382 panic("pmap_allocpagedir: have pt already\n"); 1383 } 1384 #endif /* DIAGNOSTIC */ 1385 pmap->pm_vptpt = uvm_km_zalloc(kernel_map, NBPG); 1386 if (pmap->pm_vptpt == 0) { 1387 pmap_freepagedir(pmap); 1388 return(ENOMEM); 1389 } 1390 1391 /* need to lock this all up for growkernel */ 1392 simple_lock(&pmaps_lock); 1393 /* wish we didn't have to keep this locked... */ 1394 1395 /* Duplicate the kernel mappings. */ 1396 bcopy((char *)pmap_kernel()->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1397 (char *)pmap->pm_pdir + (L1_TABLE_SIZE - KERNEL_PD_SIZE), 1398 KERNEL_PD_SIZE); 1399 1400 pte = vtopte(pmap->pm_vptpt); 1401 pmap->pm_pptpt = l2pte_pa(*pte); 1402 1403 /* Revoke cacheability and bufferability */ 1404 /* XXX should be done better than this */ 1405 *pte &= ~(L2_C | L2_B); 1406 1407 /* Wire in this page table */ 1408 pmap_map_in_l1(pmap, PTE_BASE, pmap->pm_pptpt, TRUE); 1409 1410 pt->pt_flags &= ~PTFLAG_CLEAN; /* L1 is dirty now */ 1411 1412 /* 1413 * Map the kernel page tables into the new PT map. 1414 */ 1415 bcopy((char *)(PTE_BASE 1416 + (PTE_BASE >> (PGSHIFT - 2)) 1417 + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2)), 1418 (char *)pmap->pm_vptpt + ((L1_TABLE_SIZE - KERNEL_PD_SIZE) >> 2), 1419 (KERNEL_PD_SIZE >> 2)); 1420 1421 LIST_INSERT_HEAD(&pmaps, pmap, pm_list); 1422 simple_unlock(&pmaps_lock); 1423 1424 return(0); 1425 } 1426 1427 1428 /* 1429 * Initialize a preallocated and zeroed pmap structure, 1430 * such as one in a vmspace structure. 1431 */ 1432 1433 void 1434 pmap_pinit(struct pmap *pmap) 1435 { 1436 int backoff = 6; 1437 int retry = 10; 1438 1439 PDEBUG(0, printf("pmap_pinit(%p)\n", pmap)); 1440 1441 /* Keep looping until we succeed in allocating a page directory */ 1442 while (pmap_allocpagedir(pmap) != 0) { 1443 /* 1444 * Ok we failed to allocate a suitable block of memory for an 1445 * L1 page table. This means that either: 1446 * 1. 16KB of virtual address space could not be allocated 1447 * 2. 16KB of physically contiguous memory on a 16KB boundary 1448 * could not be allocated. 1449 * 1450 * Since we cannot fail we will sleep for a while and try 1451 * again. 1452 * 1453 * Searching for a suitable L1 PT is expensive: 1454 * to avoid hogging the system when memory is really 1455 * scarce, use an exponential back-off so that 1456 * eventually we won't retry more than once every 8 1457 * seconds. This should allow other processes to run 1458 * to completion and free up resources. 1459 */ 1460 (void) ltsleep(&lbolt, PVM, "l1ptwait", (hz << 3) >> backoff, 1461 NULL); 1462 if (--retry == 0) { 1463 retry = 10; 1464 if (backoff) 1465 --backoff; 1466 } 1467 } 1468 1469 if (vector_page < KERNEL_BASE) { 1470 /* 1471 * Map the vector page. This will also allocate and map 1472 * an L2 table for it. 1473 */ 1474 pmap_enter(pmap, vector_page, systempage.pv_pa, 1475 VM_PROT_READ, VM_PROT_READ | PMAP_WIRED); 1476 pmap_update(pmap); 1477 } 1478 } 1479 1480 1481 void 1482 pmap_freepagedir(struct pmap *pmap) 1483 { 1484 /* Free the memory used for the page table mapping */ 1485 if (pmap->pm_vptpt != 0) 1486 uvm_km_free(kernel_map, (vaddr_t)pmap->pm_vptpt, NBPG); 1487 1488 /* junk the L1 page table */ 1489 if (pmap->pm_l1pt->pt_flags & PTFLAG_STATIC) { 1490 /* Add the page table to the queue */ 1491 SIMPLEQ_INSERT_TAIL(&l1pt_static_queue, pmap->pm_l1pt, pt_queue); 1492 ++l1pt_static_queue_count; 1493 } else if (l1pt_queue_count < 8) { 1494 /* Add the page table to the queue */ 1495 SIMPLEQ_INSERT_TAIL(&l1pt_queue, pmap->pm_l1pt, pt_queue); 1496 ++l1pt_queue_count; 1497 } else 1498 pmap_free_l1pt(pmap->pm_l1pt); 1499 } 1500 1501 1502 /* 1503 * Retire the given physical map from service. 1504 * Should only be called if the map contains no valid mappings. 1505 */ 1506 1507 void 1508 pmap_destroy(struct pmap *pmap) 1509 { 1510 struct vm_page *page; 1511 int count; 1512 1513 if (pmap == NULL) 1514 return; 1515 1516 PDEBUG(0, printf("pmap_destroy(%p)\n", pmap)); 1517 1518 /* 1519 * Drop reference count 1520 */ 1521 simple_lock(&pmap->pm_obj.vmobjlock); 1522 count = --pmap->pm_obj.uo_refs; 1523 simple_unlock(&pmap->pm_obj.vmobjlock); 1524 if (count > 0) { 1525 return; 1526 } 1527 1528 /* 1529 * reference count is zero, free pmap resources and then free pmap. 1530 */ 1531 1532 /* 1533 * remove it from global list of pmaps 1534 */ 1535 1536 simple_lock(&pmaps_lock); 1537 LIST_REMOVE(pmap, pm_list); 1538 simple_unlock(&pmaps_lock); 1539 1540 if (vector_page < KERNEL_BASE) { 1541 /* Remove the vector page mapping */ 1542 pmap_remove(pmap, vector_page, vector_page + NBPG); 1543 pmap_update(pmap); 1544 } 1545 1546 /* 1547 * Free any page tables still mapped 1548 * This is only temporay until pmap_enter can count the number 1549 * of mappings made in a page table. Then pmap_remove() can 1550 * reduce the count and free the pagetable when the count 1551 * reaches zero. Note that entries in this list should match the 1552 * contents of the ptpt, however this is faster than walking a 1024 1553 * entries looking for pt's 1554 * taken from i386 pmap.c 1555 */ 1556 while ((page = TAILQ_FIRST(&pmap->pm_obj.memq)) != NULL) { 1557 KASSERT((page->flags & PG_BUSY) == 0); 1558 page->wire_count = 0; 1559 uvm_pagefree(page); 1560 } 1561 1562 /* Free the page dir */ 1563 pmap_freepagedir(pmap); 1564 1565 /* return the pmap to the pool */ 1566 pool_put(&pmap_pmap_pool, pmap); 1567 } 1568 1569 1570 /* 1571 * void pmap_reference(struct pmap *pmap) 1572 * 1573 * Add a reference to the specified pmap. 1574 */ 1575 1576 void 1577 pmap_reference(struct pmap *pmap) 1578 { 1579 if (pmap == NULL) 1580 return; 1581 1582 simple_lock(&pmap->pm_lock); 1583 pmap->pm_obj.uo_refs++; 1584 simple_unlock(&pmap->pm_lock); 1585 } 1586 1587 /* 1588 * void pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1589 * 1590 * Return the start and end addresses of the kernel's virtual space. 1591 * These values are setup in pmap_bootstrap and are updated as pages 1592 * are allocated. 1593 */ 1594 1595 void 1596 pmap_virtual_space(vaddr_t *start, vaddr_t *end) 1597 { 1598 *start = virtual_avail; 1599 *end = virtual_end; 1600 } 1601 1602 /* 1603 * Activate the address space for the specified process. If the process 1604 * is the current process, load the new MMU context. 1605 */ 1606 void 1607 pmap_activate(struct proc *p) 1608 { 1609 struct pmap *pmap = p->p_vmspace->vm_map.pmap; 1610 struct pcb *pcb = &p->p_addr->u_pcb; 1611 1612 (void) pmap_extract(pmap_kernel(), (vaddr_t)pmap->pm_pdir, 1613 (paddr_t *)&pcb->pcb_pagedir); 1614 1615 PDEBUG(0, printf("pmap_activate: p=%p pmap=%p pcb=%p pdir=%p l1=%p\n", 1616 p, pmap, pcb, pmap->pm_pdir, pcb->pcb_pagedir)); 1617 1618 if (p == curproc) { 1619 PDEBUG(0, printf("pmap_activate: setting TTB\n")); 1620 setttb((u_int)pcb->pcb_pagedir); 1621 } 1622 } 1623 1624 /* 1625 * Deactivate the address space of the specified process. 1626 */ 1627 void 1628 pmap_deactivate(struct proc *p) 1629 { 1630 } 1631 1632 /* 1633 * Perform any deferred pmap operations. 1634 */ 1635 void 1636 pmap_update(struct pmap *pmap) 1637 { 1638 1639 /* 1640 * We haven't deferred any pmap operations, but we do need to 1641 * make sure TLB/cache operations have completed. 1642 */ 1643 cpu_cpwait(); 1644 } 1645 1646 /* 1647 * pmap_clean_page() 1648 * 1649 * This is a local function used to work out the best strategy to clean 1650 * a single page referenced by its entry in the PV table. It's used by 1651 * pmap_copy_page, pmap_zero page and maybe some others later on. 1652 * 1653 * Its policy is effectively: 1654 * o If there are no mappings, we don't bother doing anything with the cache. 1655 * o If there is one mapping, we clean just that page. 1656 * o If there are multiple mappings, we clean the entire cache. 1657 * 1658 * So that some functions can be further optimised, it returns 0 if it didn't 1659 * clean the entire cache, or 1 if it did. 1660 * 1661 * XXX One bug in this routine is that if the pv_entry has a single page 1662 * mapped at 0x00000000 a whole cache clean will be performed rather than 1663 * just the 1 page. Since this should not occur in everyday use and if it does 1664 * it will just result in not the most efficient clean for the page. 1665 */ 1666 static int 1667 pmap_clean_page(struct pv_entry *pv, boolean_t is_src) 1668 { 1669 struct pmap *pmap; 1670 struct pv_entry *npv; 1671 int cache_needs_cleaning = 0; 1672 vaddr_t page_to_clean = 0; 1673 1674 if (pv == NULL) 1675 /* nothing mapped in so nothing to flush */ 1676 return (0); 1677 1678 /* Since we flush the cache each time we change curproc, we 1679 * only need to flush the page if it is in the current pmap. 1680 */ 1681 if (curproc) 1682 pmap = curproc->p_vmspace->vm_map.pmap; 1683 else 1684 pmap = pmap_kernel(); 1685 1686 for (npv = pv; npv; npv = npv->pv_next) { 1687 if (npv->pv_pmap == pmap) { 1688 /* The page is mapped non-cacheable in 1689 * this map. No need to flush the cache. 1690 */ 1691 if (npv->pv_flags & PVF_NC) { 1692 #ifdef DIAGNOSTIC 1693 if (cache_needs_cleaning) 1694 panic("pmap_clean_page: " 1695 "cache inconsistency"); 1696 #endif 1697 break; 1698 } 1699 #if 0 1700 /* This doesn't work, because pmap_protect 1701 doesn't flush changes on pages that it 1702 has write-protected. */ 1703 1704 /* If the page is not writable and this 1705 is the source, then there is no need 1706 to flush it from the cache. */ 1707 else if (is_src && ! (npv->pv_flags & PVF_WRITE)) 1708 continue; 1709 #endif 1710 if (cache_needs_cleaning){ 1711 page_to_clean = 0; 1712 break; 1713 } 1714 else 1715 page_to_clean = npv->pv_va; 1716 cache_needs_cleaning = 1; 1717 } 1718 } 1719 1720 if (page_to_clean) 1721 cpu_idcache_wbinv_range(page_to_clean, NBPG); 1722 else if (cache_needs_cleaning) { 1723 cpu_idcache_wbinv_all(); 1724 return (1); 1725 } 1726 return (0); 1727 } 1728 1729 /* 1730 * pmap_zero_page() 1731 * 1732 * Zero a given physical page by mapping it at a page hook point. 1733 * In doing the zero page op, the page we zero is mapped cachable, as with 1734 * StrongARM accesses to non-cached pages are non-burst making writing 1735 * _any_ bulk data very slow. 1736 */ 1737 #if ARM_MMU_GENERIC == 1 1738 void 1739 pmap_zero_page_generic(paddr_t phys) 1740 { 1741 #ifdef DEBUG 1742 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1743 1744 if (pg->mdpage.pvh_list != NULL) 1745 panic("pmap_zero_page: page has mappings"); 1746 #endif 1747 1748 KDASSERT((phys & PGOFSET) == 0); 1749 1750 /* 1751 * Hook in the page, zero it, and purge the cache for that 1752 * zeroed page. Invalidate the TLB as needed. 1753 */ 1754 *cdst_pte = L2_S_PROTO | phys | 1755 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1756 cpu_tlb_flushD_SE(cdstp); 1757 cpu_cpwait(); 1758 bzero_page(cdstp); 1759 cpu_dcache_wbinv_range(cdstp, NBPG); 1760 } 1761 #endif /* ARM_MMU_GENERIC == 1 */ 1762 1763 #if ARM_MMU_XSCALE == 1 1764 void 1765 pmap_zero_page_xscale(paddr_t phys) 1766 { 1767 #ifdef DEBUG 1768 struct vm_page *pg = PHYS_TO_VM_PAGE(phys); 1769 1770 if (pg->mdpage.pvh_list != NULL) 1771 panic("pmap_zero_page: page has mappings"); 1772 #endif 1773 1774 KDASSERT((phys & PGOFSET) == 0); 1775 1776 /* 1777 * Hook in the page, zero it, and purge the cache for that 1778 * zeroed page. Invalidate the TLB as needed. 1779 */ 1780 *cdst_pte = L2_S_PROTO | phys | 1781 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1782 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1783 cpu_tlb_flushD_SE(cdstp); 1784 cpu_cpwait(); 1785 bzero_page(cdstp); 1786 xscale_cache_clean_minidata(); 1787 } 1788 #endif /* ARM_MMU_XSCALE == 1 */ 1789 1790 /* pmap_pageidlezero() 1791 * 1792 * The same as above, except that we assume that the page is not 1793 * mapped. This means we never have to flush the cache first. Called 1794 * from the idle loop. 1795 */ 1796 boolean_t 1797 pmap_pageidlezero(paddr_t phys) 1798 { 1799 int i, *ptr; 1800 boolean_t rv = TRUE; 1801 #ifdef DEBUG 1802 struct vm_page *pg; 1803 1804 pg = PHYS_TO_VM_PAGE(phys); 1805 if (pg->mdpage.pvh_list != NULL) 1806 panic("pmap_pageidlezero: page has mappings"); 1807 #endif 1808 1809 KDASSERT((phys & PGOFSET) == 0); 1810 1811 /* 1812 * Hook in the page, zero it, and purge the cache for that 1813 * zeroed page. Invalidate the TLB as needed. 1814 */ 1815 *cdst_pte = L2_S_PROTO | phys | 1816 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1817 cpu_tlb_flushD_SE(cdstp); 1818 cpu_cpwait(); 1819 1820 for (i = 0, ptr = (int *)cdstp; 1821 i < (NBPG / sizeof(int)); i++) { 1822 if (sched_whichqs != 0) { 1823 /* 1824 * A process has become ready. Abort now, 1825 * so we don't keep it waiting while we 1826 * do slow memory access to finish this 1827 * page. 1828 */ 1829 rv = FALSE; 1830 break; 1831 } 1832 *ptr++ = 0; 1833 } 1834 1835 if (rv) 1836 /* 1837 * if we aborted we'll rezero this page again later so don't 1838 * purge it unless we finished it 1839 */ 1840 cpu_dcache_wbinv_range(cdstp, NBPG); 1841 return (rv); 1842 } 1843 1844 /* 1845 * pmap_copy_page() 1846 * 1847 * Copy one physical page into another, by mapping the pages into 1848 * hook points. The same comment regarding cachability as in 1849 * pmap_zero_page also applies here. 1850 */ 1851 #if ARM_MMU_GENERIC == 1 1852 void 1853 pmap_copy_page_generic(paddr_t src, paddr_t dst) 1854 { 1855 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1856 #ifdef DEBUG 1857 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1858 1859 if (dst_pg->mdpage.pvh_list != NULL) 1860 panic("pmap_copy_page: dst page has mappings"); 1861 #endif 1862 1863 KDASSERT((src & PGOFSET) == 0); 1864 KDASSERT((dst & PGOFSET) == 0); 1865 1866 /* 1867 * Clean the source page. Hold the source page's lock for 1868 * the duration of the copy so that no other mappings can 1869 * be created while we have a potentially aliased mapping. 1870 */ 1871 simple_lock(&src_pg->mdpage.pvh_slock); 1872 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1873 1874 /* 1875 * Map the pages into the page hook points, copy them, and purge 1876 * the cache for the appropriate page. Invalidate the TLB 1877 * as required. 1878 */ 1879 *csrc_pte = L2_S_PROTO | src | 1880 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; 1881 *cdst_pte = L2_S_PROTO | dst | 1882 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; 1883 cpu_tlb_flushD_SE(csrcp); 1884 cpu_tlb_flushD_SE(cdstp); 1885 cpu_cpwait(); 1886 bcopy_page(csrcp, cdstp); 1887 cpu_dcache_inv_range(csrcp, NBPG); 1888 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1889 cpu_dcache_wbinv_range(cdstp, NBPG); 1890 } 1891 #endif /* ARM_MMU_GENERIC == 1 */ 1892 1893 #if ARM_MMU_XSCALE == 1 1894 void 1895 pmap_copy_page_xscale(paddr_t src, paddr_t dst) 1896 { 1897 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); 1898 #ifdef DEBUG 1899 struct vm_page *dst_pg = PHYS_TO_VM_PAGE(dst); 1900 1901 if (dst_pg->mdpage.pvh_list != NULL) 1902 panic("pmap_copy_page: dst page has mappings"); 1903 #endif 1904 1905 KDASSERT((src & PGOFSET) == 0); 1906 KDASSERT((dst & PGOFSET) == 0); 1907 1908 /* 1909 * Clean the source page. Hold the source page's lock for 1910 * the duration of the copy so that no other mappings can 1911 * be created while we have a potentially aliased mapping. 1912 */ 1913 simple_lock(&src_pg->mdpage.pvh_slock); 1914 (void) pmap_clean_page(src_pg->mdpage.pvh_list, TRUE); 1915 1916 /* 1917 * Map the pages into the page hook points, copy them, and purge 1918 * the cache for the appropriate page. Invalidate the TLB 1919 * as required. 1920 */ 1921 *csrc_pte = L2_S_PROTO | src | 1922 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 1923 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1924 *cdst_pte = L2_S_PROTO | dst | 1925 L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | 1926 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ 1927 cpu_tlb_flushD_SE(csrcp); 1928 cpu_tlb_flushD_SE(cdstp); 1929 cpu_cpwait(); 1930 bcopy_page(csrcp, cdstp); 1931 simple_unlock(&src_pg->mdpage.pvh_slock); /* cache is safe again */ 1932 xscale_cache_clean_minidata(); 1933 } 1934 #endif /* ARM_MMU_XSCALE == 1 */ 1935 1936 #if 0 1937 void 1938 pmap_pte_addref(struct pmap *pmap, vaddr_t va) 1939 { 1940 pd_entry_t *pde; 1941 paddr_t pa; 1942 struct vm_page *m; 1943 1944 if (pmap == pmap_kernel()) 1945 return; 1946 1947 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1948 pa = pmap_pte_pa(pde); 1949 m = PHYS_TO_VM_PAGE(pa); 1950 ++m->wire_count; 1951 #ifdef MYCROFT_HACK 1952 printf("addref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1953 pmap, va, pde, pa, m, m->wire_count); 1954 #endif 1955 } 1956 1957 void 1958 pmap_pte_delref(struct pmap *pmap, vaddr_t va) 1959 { 1960 pd_entry_t *pde; 1961 paddr_t pa; 1962 struct vm_page *m; 1963 1964 if (pmap == pmap_kernel()) 1965 return; 1966 1967 pde = pmap_pde(pmap, va & ~(3 << L1_S_SHIFT)); 1968 pa = pmap_pte_pa(pde); 1969 m = PHYS_TO_VM_PAGE(pa); 1970 --m->wire_count; 1971 #ifdef MYCROFT_HACK 1972 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p wire=%d\n", 1973 pmap, va, pde, pa, m, m->wire_count); 1974 #endif 1975 if (m->wire_count == 0) { 1976 #ifdef MYCROFT_HACK 1977 printf("delref pmap=%p va=%08lx pde=%p pa=%08lx m=%p\n", 1978 pmap, va, pde, pa, m); 1979 #endif 1980 pmap_unmap_in_l1(pmap, va); 1981 uvm_pagefree(m); 1982 --pmap->pm_stats.resident_count; 1983 } 1984 } 1985 #else 1986 #define pmap_pte_addref(pmap, va) 1987 #define pmap_pte_delref(pmap, va) 1988 #endif 1989 1990 /* 1991 * Since we have a virtually indexed cache, we may need to inhibit caching if 1992 * there is more than one mapping and at least one of them is writable. 1993 * Since we purge the cache on every context switch, we only need to check for 1994 * other mappings within the same pmap, or kernel_pmap. 1995 * This function is also called when a page is unmapped, to possibly reenable 1996 * caching on any remaining mappings. 1997 * 1998 * The code implements the following logic, where: 1999 * 2000 * KW = # of kernel read/write pages 2001 * KR = # of kernel read only pages 2002 * UW = # of user read/write pages 2003 * UR = # of user read only pages 2004 * OW = # of user read/write pages in another pmap, then 2005 * 2006 * KC = kernel mapping is cacheable 2007 * UC = user mapping is cacheable 2008 * 2009 * KW=0,KR=0 KW=0,KR>0 KW=1,KR=0 KW>1,KR>=0 2010 * +--------------------------------------------- 2011 * UW=0,UR=0,OW=0 | --- KC=1 KC=1 KC=0 2012 * UW=0,UR>0,OW=0 | UC=1 KC=1,UC=1 KC=0,UC=0 KC=0,UC=0 2013 * UW=0,UR>0,OW>0 | UC=1 KC=0,UC=1 KC=0,UC=0 KC=0,UC=0 2014 * UW=1,UR=0,OW=0 | UC=1 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2015 * UW>1,UR>=0,OW>=0 | UC=0 KC=0,UC=0 KC=0,UC=0 KC=0,UC=0 2016 * 2017 * Note that the pmap must have it's ptes mapped in, and passed with ptes. 2018 */ 2019 __inline static void 2020 pmap_vac_me_harder(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2021 boolean_t clear_cache) 2022 { 2023 if (pmap == pmap_kernel()) 2024 pmap_vac_me_kpmap(pmap, pg, ptes, clear_cache); 2025 else 2026 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2027 } 2028 2029 static void 2030 pmap_vac_me_kpmap(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2031 boolean_t clear_cache) 2032 { 2033 int user_entries = 0; 2034 int user_writable = 0; 2035 int user_cacheable = 0; 2036 int kernel_entries = 0; 2037 int kernel_writable = 0; 2038 int kernel_cacheable = 0; 2039 struct pv_entry *pv; 2040 struct pmap *last_pmap = pmap; 2041 2042 #ifdef DIAGNOSTIC 2043 if (pmap != pmap_kernel()) 2044 panic("pmap_vac_me_kpmap: pmap != pmap_kernel()"); 2045 #endif 2046 2047 /* 2048 * Pass one, see if there are both kernel and user pmaps for 2049 * this page. Calculate whether there are user-writable or 2050 * kernel-writable pages. 2051 */ 2052 for (pv = pg->mdpage.pvh_list; pv != NULL; pv = pv->pv_next) { 2053 if (pv->pv_pmap != pmap) { 2054 user_entries++; 2055 if (pv->pv_flags & PVF_WRITE) 2056 user_writable++; 2057 if ((pv->pv_flags & PVF_NC) == 0) 2058 user_cacheable++; 2059 } else { 2060 kernel_entries++; 2061 if (pv->pv_flags & PVF_WRITE) 2062 kernel_writable++; 2063 if ((pv->pv_flags & PVF_NC) == 0) 2064 kernel_cacheable++; 2065 } 2066 } 2067 2068 /* 2069 * We know we have just been updating a kernel entry, so if 2070 * all user pages are already cacheable, then there is nothing 2071 * further to do. 2072 */ 2073 if (kernel_entries == 0 && 2074 user_cacheable == user_entries) 2075 return; 2076 2077 if (user_entries) { 2078 /* 2079 * Scan over the list again, for each entry, if it 2080 * might not be set correctly, call pmap_vac_me_user 2081 * to recalculate the settings. 2082 */ 2083 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 2084 /* 2085 * We know kernel mappings will get set 2086 * correctly in other calls. We also know 2087 * that if the pmap is the same as last_pmap 2088 * then we've just handled this entry. 2089 */ 2090 if (pv->pv_pmap == pmap || pv->pv_pmap == last_pmap) 2091 continue; 2092 /* 2093 * If there are kernel entries and this page 2094 * is writable but non-cacheable, then we can 2095 * skip this entry also. 2096 */ 2097 if (kernel_entries > 0 && 2098 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 2099 (PVF_NC | PVF_WRITE)) 2100 continue; 2101 /* 2102 * Similarly if there are no kernel-writable 2103 * entries and the page is already 2104 * read-only/cacheable. 2105 */ 2106 if (kernel_writable == 0 && 2107 (pv->pv_flags & (PVF_NC | PVF_WRITE)) == 0) 2108 continue; 2109 /* 2110 * For some of the remaining cases, we know 2111 * that we must recalculate, but for others we 2112 * can't tell if they are correct or not, so 2113 * we recalculate anyway. 2114 */ 2115 pmap_unmap_ptes(last_pmap); 2116 last_pmap = pv->pv_pmap; 2117 ptes = pmap_map_ptes(last_pmap); 2118 pmap_vac_me_user(last_pmap, pg, ptes, 2119 pmap_is_curpmap(last_pmap)); 2120 } 2121 /* Restore the pte mapping that was passed to us. */ 2122 if (last_pmap != pmap) { 2123 pmap_unmap_ptes(last_pmap); 2124 ptes = pmap_map_ptes(pmap); 2125 } 2126 if (kernel_entries == 0) 2127 return; 2128 } 2129 2130 pmap_vac_me_user(pmap, pg, ptes, clear_cache); 2131 return; 2132 } 2133 2134 static void 2135 pmap_vac_me_user(struct pmap *pmap, struct vm_page *pg, pt_entry_t *ptes, 2136 boolean_t clear_cache) 2137 { 2138 struct pmap *kpmap = pmap_kernel(); 2139 struct pv_entry *pv, *npv; 2140 int entries = 0; 2141 int writable = 0; 2142 int cacheable_entries = 0; 2143 int kern_cacheable = 0; 2144 int other_writable = 0; 2145 2146 pv = pg->mdpage.pvh_list; 2147 KASSERT(ptes != NULL); 2148 2149 /* 2150 * Count mappings and writable mappings in this pmap. 2151 * Include kernel mappings as part of our own. 2152 * Keep a pointer to the first one. 2153 */ 2154 for (npv = pv; npv; npv = npv->pv_next) { 2155 /* Count mappings in the same pmap */ 2156 if (pmap == npv->pv_pmap || 2157 kpmap == npv->pv_pmap) { 2158 if (entries++ == 0) 2159 pv = npv; 2160 /* Cacheable mappings */ 2161 if ((npv->pv_flags & PVF_NC) == 0) { 2162 cacheable_entries++; 2163 if (kpmap == npv->pv_pmap) 2164 kern_cacheable++; 2165 } 2166 /* Writable mappings */ 2167 if (npv->pv_flags & PVF_WRITE) 2168 ++writable; 2169 } else if (npv->pv_flags & PVF_WRITE) 2170 other_writable = 1; 2171 } 2172 2173 PDEBUG(3,printf("pmap_vac_me_harder: pmap %p Entries %d, " 2174 "writable %d cacheable %d %s\n", pmap, entries, writable, 2175 cacheable_entries, clear_cache ? "clean" : "no clean")); 2176 2177 /* 2178 * Enable or disable caching as necessary. 2179 * Note: the first entry might be part of the kernel pmap, 2180 * so we can't assume this is indicative of the state of the 2181 * other (maybe non-kpmap) entries. 2182 */ 2183 if ((entries > 1 && writable) || 2184 (entries > 0 && pmap == kpmap && other_writable)) { 2185 if (cacheable_entries == 0) 2186 return; 2187 for (npv = pv; npv; npv = npv->pv_next) { 2188 if ((pmap == npv->pv_pmap 2189 || kpmap == npv->pv_pmap) && 2190 (npv->pv_flags & PVF_NC) == 0) { 2191 ptes[arm_btop(npv->pv_va)] &= ~(L2_C | L2_B); 2192 npv->pv_flags |= PVF_NC; 2193 /* 2194 * If this page needs flushing from the 2195 * cache, and we aren't going to do it 2196 * below, do it now. 2197 */ 2198 if ((cacheable_entries < 4 && 2199 (clear_cache || npv->pv_pmap == kpmap)) || 2200 (npv->pv_pmap == kpmap && 2201 !clear_cache && kern_cacheable < 4)) { 2202 cpu_idcache_wbinv_range(npv->pv_va, 2203 NBPG); 2204 cpu_tlb_flushID_SE(npv->pv_va); 2205 } 2206 } 2207 } 2208 if ((clear_cache && cacheable_entries >= 4) || 2209 kern_cacheable >= 4) { 2210 cpu_idcache_wbinv_all(); 2211 cpu_tlb_flushID(); 2212 } 2213 cpu_cpwait(); 2214 } else if (entries > 0) { 2215 /* 2216 * Turn cacheing back on for some pages. If it is a kernel 2217 * page, only do so if there are no other writable pages. 2218 */ 2219 for (npv = pv; npv; npv = npv->pv_next) { 2220 if ((pmap == npv->pv_pmap || 2221 (kpmap == npv->pv_pmap && other_writable == 0)) && 2222 (npv->pv_flags & PVF_NC)) { 2223 ptes[arm_btop(npv->pv_va)] |= 2224 pte_l2_s_cache_mode; 2225 npv->pv_flags &= ~PVF_NC; 2226 } 2227 } 2228 } 2229 } 2230 2231 /* 2232 * pmap_remove() 2233 * 2234 * pmap_remove is responsible for nuking a number of mappings for a range 2235 * of virtual address space in the current pmap. To do this efficiently 2236 * is interesting, because in a number of cases a wide virtual address 2237 * range may be supplied that contains few actual mappings. So, the 2238 * optimisations are: 2239 * 1. Try and skip over hunks of address space for which an L1 entry 2240 * does not exist. 2241 * 2. Build up a list of pages we've hit, up to a maximum, so we can 2242 * maybe do just a partial cache clean. This path of execution is 2243 * complicated by the fact that the cache must be flushed _before_ 2244 * the PTE is nuked, being a VAC :-) 2245 * 3. Maybe later fast-case a single page, but I don't think this is 2246 * going to make _that_ much difference overall. 2247 */ 2248 2249 #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 2250 2251 void 2252 pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva) 2253 { 2254 int cleanlist_idx = 0; 2255 struct pagelist { 2256 vaddr_t va; 2257 pt_entry_t *pte; 2258 } cleanlist[PMAP_REMOVE_CLEAN_LIST_SIZE]; 2259 pt_entry_t *pte = 0, *ptes; 2260 paddr_t pa; 2261 int pmap_active; 2262 struct vm_page *pg; 2263 2264 /* Exit quick if there is no pmap */ 2265 if (!pmap) 2266 return; 2267 2268 PDEBUG(0, printf("pmap_remove: pmap=%p sva=%08lx eva=%08lx\n", 2269 pmap, sva, eva)); 2270 2271 /* 2272 * we lock in the pmap => vm_page direction 2273 */ 2274 PMAP_MAP_TO_HEAD_LOCK(); 2275 2276 ptes = pmap_map_ptes(pmap); 2277 /* Get a page table pointer */ 2278 while (sva < eva) { 2279 if (pmap_pde_page(pmap_pde(pmap, sva))) 2280 break; 2281 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2282 } 2283 2284 pte = &ptes[arm_btop(sva)]; 2285 /* Note if the pmap is active thus require cache and tlb cleans */ 2286 pmap_active = pmap_is_curpmap(pmap); 2287 2288 /* Now loop along */ 2289 while (sva < eva) { 2290 /* Check if we can move to the next PDE (l1 chunk) */ 2291 if (!(sva & L2_ADDR_BITS)) 2292 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2293 sva += L1_S_SIZE; 2294 pte += arm_btop(L1_S_SIZE); 2295 continue; 2296 } 2297 2298 /* We've found a valid PTE, so this page of PTEs has to go. */ 2299 if (pmap_pte_v(pte)) { 2300 /* Update statistics */ 2301 --pmap->pm_stats.resident_count; 2302 2303 /* 2304 * Add this page to our cache remove list, if we can. 2305 * If, however the cache remove list is totally full, 2306 * then do a complete cache invalidation taking note 2307 * to backtrack the PTE table beforehand, and ignore 2308 * the lists in future because there's no longer any 2309 * point in bothering with them (we've paid the 2310 * penalty, so will carry on unhindered). Otherwise, 2311 * when we fall out, we just clean the list. 2312 */ 2313 PDEBUG(10, printf("remove: inv pte at %p(%x) ", pte, *pte)); 2314 pa = pmap_pte_pa(pte); 2315 2316 if (cleanlist_idx < PMAP_REMOVE_CLEAN_LIST_SIZE) { 2317 /* Add to the clean list. */ 2318 cleanlist[cleanlist_idx].pte = pte; 2319 cleanlist[cleanlist_idx].va = sva; 2320 cleanlist_idx++; 2321 } else if (cleanlist_idx == PMAP_REMOVE_CLEAN_LIST_SIZE) { 2322 int cnt; 2323 2324 /* Nuke everything if needed. */ 2325 if (pmap_active) { 2326 cpu_idcache_wbinv_all(); 2327 cpu_tlb_flushID(); 2328 } 2329 2330 /* 2331 * Roll back the previous PTE list, 2332 * and zero out the current PTE. 2333 */ 2334 for (cnt = 0; cnt < PMAP_REMOVE_CLEAN_LIST_SIZE; cnt++) { 2335 *cleanlist[cnt].pte = 0; 2336 pmap_pte_delref(pmap, cleanlist[cnt].va); 2337 } 2338 *pte = 0; 2339 pmap_pte_delref(pmap, sva); 2340 cleanlist_idx++; 2341 } else { 2342 /* 2343 * We've already nuked the cache and 2344 * TLB, so just carry on regardless, 2345 * and we won't need to do it again 2346 */ 2347 *pte = 0; 2348 pmap_pte_delref(pmap, sva); 2349 } 2350 2351 /* 2352 * Update flags. In a number of circumstances, 2353 * we could cluster a lot of these and do a 2354 * number of sequential pages in one go. 2355 */ 2356 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2357 struct pv_entry *pve; 2358 simple_lock(&pg->mdpage.pvh_slock); 2359 pve = pmap_remove_pv(pg, pmap, sva); 2360 pmap_free_pv(pmap, pve); 2361 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2362 simple_unlock(&pg->mdpage.pvh_slock); 2363 } 2364 } 2365 sva += NBPG; 2366 pte++; 2367 } 2368 2369 pmap_unmap_ptes(pmap); 2370 /* 2371 * Now, if we've fallen through down to here, chances are that there 2372 * are less than PMAP_REMOVE_CLEAN_LIST_SIZE mappings left. 2373 */ 2374 if (cleanlist_idx <= PMAP_REMOVE_CLEAN_LIST_SIZE) { 2375 u_int cnt; 2376 2377 for (cnt = 0; cnt < cleanlist_idx; cnt++) { 2378 if (pmap_active) { 2379 cpu_idcache_wbinv_range(cleanlist[cnt].va, 2380 NBPG); 2381 *cleanlist[cnt].pte = 0; 2382 cpu_tlb_flushID_SE(cleanlist[cnt].va); 2383 } else 2384 *cleanlist[cnt].pte = 0; 2385 pmap_pte_delref(pmap, cleanlist[cnt].va); 2386 } 2387 } 2388 PMAP_MAP_TO_HEAD_UNLOCK(); 2389 } 2390 2391 /* 2392 * Routine: pmap_remove_all 2393 * Function: 2394 * Removes this physical page from 2395 * all physical maps in which it resides. 2396 * Reflects back modify bits to the pager. 2397 */ 2398 2399 static void 2400 pmap_remove_all(struct vm_page *pg) 2401 { 2402 struct pv_entry *pv, *npv; 2403 struct pmap *pmap; 2404 pt_entry_t *pte, *ptes; 2405 2406 PDEBUG(0, printf("pmap_remove_all: pa=%lx ", VM_PAGE_TO_PHYS(pg))); 2407 2408 /* set vm_page => pmap locking */ 2409 PMAP_HEAD_TO_MAP_LOCK(); 2410 2411 simple_lock(&pg->mdpage.pvh_slock); 2412 2413 pv = pg->mdpage.pvh_list; 2414 if (pv == NULL) { 2415 PDEBUG(0, printf("free page\n")); 2416 simple_unlock(&pg->mdpage.pvh_slock); 2417 PMAP_HEAD_TO_MAP_UNLOCK(); 2418 return; 2419 } 2420 pmap_clean_page(pv, FALSE); 2421 2422 while (pv) { 2423 pmap = pv->pv_pmap; 2424 ptes = pmap_map_ptes(pmap); 2425 pte = &ptes[arm_btop(pv->pv_va)]; 2426 2427 PDEBUG(0, printf("[%p,%08x,%08lx,%08x] ", pmap, *pte, 2428 pv->pv_va, pv->pv_flags)); 2429 #ifdef DEBUG 2430 if (pmap_pde_page(pmap_pde(pmap, pv->pv_va)) == 0 || 2431 pmap_pte_v(pte) == 0 || 2432 pmap_pte_pa(pte) != VM_PAGE_TO_PHYS(pg)) 2433 panic("pmap_remove_all: bad mapping"); 2434 #endif /* DEBUG */ 2435 2436 /* 2437 * Update statistics 2438 */ 2439 --pmap->pm_stats.resident_count; 2440 2441 /* Wired bit */ 2442 if (pv->pv_flags & PVF_WIRED) 2443 --pmap->pm_stats.wired_count; 2444 2445 /* 2446 * Invalidate the PTEs. 2447 * XXX: should cluster them up and invalidate as many 2448 * as possible at once. 2449 */ 2450 2451 #ifdef needednotdone 2452 reduce wiring count on page table pages as references drop 2453 #endif 2454 2455 *pte = 0; 2456 pmap_pte_delref(pmap, pv->pv_va); 2457 2458 npv = pv->pv_next; 2459 pmap_free_pv(pmap, pv); 2460 pv = npv; 2461 pmap_unmap_ptes(pmap); 2462 } 2463 pg->mdpage.pvh_list = NULL; 2464 simple_unlock(&pg->mdpage.pvh_slock); 2465 PMAP_HEAD_TO_MAP_UNLOCK(); 2466 2467 PDEBUG(0, printf("done\n")); 2468 cpu_tlb_flushID(); 2469 cpu_cpwait(); 2470 } 2471 2472 2473 /* 2474 * Set the physical protection on the specified range of this map as requested. 2475 */ 2476 2477 void 2478 pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 2479 { 2480 pt_entry_t *pte = NULL, *ptes; 2481 struct vm_page *pg; 2482 int armprot; 2483 int flush = 0; 2484 paddr_t pa; 2485 2486 PDEBUG(0, printf("pmap_protect: pmap=%p %08lx->%08lx %x\n", 2487 pmap, sva, eva, prot)); 2488 2489 if (~prot & VM_PROT_READ) { 2490 /* Just remove the mappings. */ 2491 pmap_remove(pmap, sva, eva); 2492 /* pmap_update not needed as it should be called by the caller 2493 * of pmap_protect */ 2494 return; 2495 } 2496 if (prot & VM_PROT_WRITE) { 2497 /* 2498 * If this is a read->write transition, just ignore it and let 2499 * uvm_fault() take care of it later. 2500 */ 2501 return; 2502 } 2503 2504 /* Need to lock map->head */ 2505 PMAP_MAP_TO_HEAD_LOCK(); 2506 2507 ptes = pmap_map_ptes(pmap); 2508 /* 2509 * We need to acquire a pointer to a page table page before entering 2510 * the following loop. 2511 */ 2512 while (sva < eva) { 2513 if (pmap_pde_page(pmap_pde(pmap, sva))) 2514 break; 2515 sva = (sva & L1_S_FRAME) + L1_S_SIZE; 2516 } 2517 2518 pte = &ptes[arm_btop(sva)]; 2519 2520 while (sva < eva) { 2521 /* only check once in a while */ 2522 if ((sva & L2_ADDR_BITS) == 0) { 2523 if (!pmap_pde_page(pmap_pde(pmap, sva))) { 2524 /* We can race ahead here, to the next pde. */ 2525 sva += L1_S_SIZE; 2526 pte += arm_btop(L1_S_SIZE); 2527 continue; 2528 } 2529 } 2530 2531 if (!pmap_pte_v(pte)) 2532 goto next; 2533 2534 flush = 1; 2535 2536 armprot = 0; 2537 if (sva < VM_MAXUSER_ADDRESS) 2538 armprot |= L2_S_PROT_U; 2539 else if (sva < VM_MAX_ADDRESS) 2540 armprot |= L2_S_PROT_W; /* XXX Ekk what is this ? */ 2541 *pte = (*pte & 0xfffff00f) | armprot; 2542 2543 pa = pmap_pte_pa(pte); 2544 2545 /* Get the physical page index */ 2546 2547 /* Clear write flag */ 2548 if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { 2549 simple_lock(&pg->mdpage.pvh_slock); 2550 (void) pmap_modify_pv(pmap, sva, pg, PVF_WRITE, 0); 2551 pmap_vac_me_harder(pmap, pg, ptes, FALSE); 2552 simple_unlock(&pg->mdpage.pvh_slock); 2553 } 2554 2555 next: 2556 sva += NBPG; 2557 pte++; 2558 } 2559 pmap_unmap_ptes(pmap); 2560 PMAP_MAP_TO_HEAD_UNLOCK(); 2561 if (flush) 2562 cpu_tlb_flushID(); 2563 } 2564 2565 /* 2566 * void pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2567 * int flags) 2568 * 2569 * Insert the given physical page (p) at 2570 * the specified virtual address (v) in the 2571 * target physical map with the protection requested. 2572 * 2573 * If specified, the page will be wired down, meaning 2574 * that the related pte can not be reclaimed. 2575 * 2576 * NB: This is the only routine which MAY NOT lazy-evaluate 2577 * or lose information. That is, this routine must actually 2578 * insert this page into the given map NOW. 2579 */ 2580 2581 int 2582 pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, 2583 int flags) 2584 { 2585 pt_entry_t *ptes, opte, npte; 2586 paddr_t opa; 2587 boolean_t wired = (flags & PMAP_WIRED) != 0; 2588 struct vm_page *pg; 2589 struct pv_entry *pve; 2590 int error, nflags; 2591 2592 PDEBUG(5, printf("pmap_enter: V%08lx P%08lx in pmap %p prot=%08x, wired = %d\n", 2593 va, pa, pmap, prot, wired)); 2594 2595 #ifdef DIAGNOSTIC 2596 /* Valid address ? */ 2597 if (va >= (pmap_curmaxkvaddr)) 2598 panic("pmap_enter: too big"); 2599 if (pmap != pmap_kernel() && va != 0) { 2600 if (va < VM_MIN_ADDRESS || va >= VM_MAXUSER_ADDRESS) 2601 panic("pmap_enter: kernel page in user map"); 2602 } else { 2603 if (va >= VM_MIN_ADDRESS && va < VM_MAXUSER_ADDRESS) 2604 panic("pmap_enter: user page in kernel map"); 2605 if (va >= VM_MAXUSER_ADDRESS && va < VM_MAX_ADDRESS) 2606 panic("pmap_enter: entering PT page"); 2607 } 2608 #endif 2609 2610 KDASSERT(((va | pa) & PGOFSET) == 0); 2611 2612 /* 2613 * Get a pointer to the page. Later on in this function, we 2614 * test for a managed page by checking pg != NULL. 2615 */ 2616 pg = pmap_initialized ? PHYS_TO_VM_PAGE(pa) : NULL; 2617 2618 /* get lock */ 2619 PMAP_MAP_TO_HEAD_LOCK(); 2620 2621 /* 2622 * map the ptes. If there's not already an L2 table for this 2623 * address, allocate one. 2624 */ 2625 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2626 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 2627 struct vm_page *ptp; 2628 2629 /* kernel should be pre-grown */ 2630 KASSERT(pmap != pmap_kernel()); 2631 2632 /* if failure is allowed then don't try too hard */ 2633 ptp = pmap_get_ptp(pmap, va & L1_S_FRAME); 2634 if (ptp == NULL) { 2635 if (flags & PMAP_CANFAIL) { 2636 error = ENOMEM; 2637 goto out; 2638 } 2639 panic("pmap_enter: get ptp failed"); 2640 } 2641 } 2642 opte = ptes[arm_btop(va)]; 2643 2644 nflags = 0; 2645 if (prot & VM_PROT_WRITE) 2646 nflags |= PVF_WRITE; 2647 if (wired) 2648 nflags |= PVF_WIRED; 2649 2650 /* Is the pte valid ? If so then this page is already mapped */ 2651 if (l2pte_valid(opte)) { 2652 /* Get the physical address of the current page mapped */ 2653 opa = l2pte_pa(opte); 2654 2655 /* Are we mapping the same page ? */ 2656 if (opa == pa) { 2657 /* Has the wiring changed ? */ 2658 if (pg != NULL) { 2659 simple_lock(&pg->mdpage.pvh_slock); 2660 (void) pmap_modify_pv(pmap, va, pg, 2661 PVF_WRITE | PVF_WIRED, nflags); 2662 simple_unlock(&pg->mdpage.pvh_slock); 2663 } 2664 } else { 2665 struct vm_page *opg; 2666 2667 /* We are replacing the page with a new one. */ 2668 cpu_idcache_wbinv_range(va, NBPG); 2669 2670 /* 2671 * If it is part of our managed memory then we 2672 * must remove it from the PV list 2673 */ 2674 if ((opg = PHYS_TO_VM_PAGE(opa)) != NULL) { 2675 simple_lock(&opg->mdpage.pvh_slock); 2676 pve = pmap_remove_pv(opg, pmap, va); 2677 simple_unlock(&opg->mdpage.pvh_slock); 2678 } else { 2679 pve = NULL; 2680 } 2681 2682 goto enter; 2683 } 2684 } else { 2685 opa = 0; 2686 pve = NULL; 2687 pmap_pte_addref(pmap, va); 2688 2689 /* pte is not valid so we must be hooking in a new page */ 2690 ++pmap->pm_stats.resident_count; 2691 2692 enter: 2693 /* 2694 * Enter on the PV list if part of our managed memory 2695 */ 2696 if (pg != NULL) { 2697 if (pve == NULL) { 2698 pve = pmap_alloc_pv(pmap, ALLOCPV_NEED); 2699 if (pve == NULL) { 2700 if (flags & PMAP_CANFAIL) { 2701 error = ENOMEM; 2702 goto out; 2703 } 2704 panic("pmap_enter: no pv entries " 2705 "available"); 2706 } 2707 } 2708 /* enter_pv locks pvh when adding */ 2709 pmap_enter_pv(pg, pve, pmap, va, NULL, nflags); 2710 } else { 2711 if (pve != NULL) 2712 pmap_free_pv(pmap, pve); 2713 } 2714 } 2715 2716 /* Construct the pte, giving the correct access. */ 2717 npte = pa; 2718 2719 /* VA 0 is magic. */ 2720 if (pmap != pmap_kernel() && va != vector_page) 2721 npte |= L2_S_PROT_U; 2722 2723 if (pg != NULL) { 2724 #ifdef DIAGNOSTIC 2725 if ((flags & VM_PROT_ALL) & ~prot) 2726 panic("pmap_enter: access_type exceeds prot"); 2727 #endif 2728 npte |= pte_l2_s_cache_mode; 2729 if (flags & VM_PROT_WRITE) { 2730 npte |= L2_S_PROTO | L2_S_PROT_W; 2731 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 2732 } else if (flags & VM_PROT_ALL) { 2733 npte |= L2_S_PROTO; 2734 pg->mdpage.pvh_attrs |= PVF_REF; 2735 } else 2736 npte |= L2_TYPE_INV; 2737 } else { 2738 if (prot & VM_PROT_WRITE) 2739 npte |= L2_S_PROTO | L2_S_PROT_W; 2740 else if (prot & VM_PROT_ALL) 2741 npte |= L2_S_PROTO; 2742 else 2743 npte |= L2_TYPE_INV; 2744 } 2745 2746 ptes[arm_btop(va)] = npte; 2747 2748 if (pg != NULL) { 2749 simple_lock(&pg->mdpage.pvh_slock); 2750 pmap_vac_me_harder(pmap, pg, ptes, pmap_is_curpmap(pmap)); 2751 simple_unlock(&pg->mdpage.pvh_slock); 2752 } 2753 2754 /* Better flush the TLB ... */ 2755 cpu_tlb_flushID_SE(va); 2756 error = 0; 2757 out: 2758 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2759 PMAP_MAP_TO_HEAD_UNLOCK(); 2760 2761 return error; 2762 } 2763 2764 /* 2765 * pmap_kenter_pa: enter a kernel mapping 2766 * 2767 * => no need to lock anything assume va is already allocated 2768 * => should be faster than normal pmap enter function 2769 */ 2770 void 2771 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot) 2772 { 2773 pt_entry_t *pte; 2774 2775 pte = vtopte(va); 2776 KASSERT(!pmap_pte_v(pte)); 2777 2778 /* XXX r/w! */ 2779 *pte = L2_S_PROTO | pa | 2780 L2_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | 2781 pte_l2_s_cache_mode; 2782 } 2783 2784 void 2785 pmap_kremove(vaddr_t va, vsize_t len) 2786 { 2787 pt_entry_t *pte; 2788 2789 for (len >>= PAGE_SHIFT; len > 0; len--, va += PAGE_SIZE) { 2790 2791 /* 2792 * We assume that we will only be called with small 2793 * regions of memory. 2794 */ 2795 2796 KASSERT(pmap_pde_page(pmap_pde(pmap_kernel(), va))); 2797 pte = vtopte(va); 2798 cpu_idcache_wbinv_range(va, PAGE_SIZE); 2799 *pte = 0; 2800 cpu_tlb_flushID_SE(va); 2801 } 2802 } 2803 2804 /* 2805 * pmap_page_protect: 2806 * 2807 * Lower the permission for all mappings to a given page. 2808 */ 2809 2810 void 2811 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2812 { 2813 2814 PDEBUG(0, printf("pmap_page_protect(pa=%lx, prot=%d)\n", 2815 VM_PAGE_TO_PHYS(pg), prot)); 2816 2817 switch(prot) { 2818 case VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE: 2819 case VM_PROT_READ|VM_PROT_WRITE: 2820 return; 2821 2822 case VM_PROT_READ: 2823 case VM_PROT_READ|VM_PROT_EXECUTE: 2824 pmap_clearbit(pg, PVF_WRITE); 2825 break; 2826 2827 default: 2828 pmap_remove_all(pg); 2829 break; 2830 } 2831 } 2832 2833 2834 /* 2835 * Routine: pmap_unwire 2836 * Function: Clear the wired attribute for a map/virtual-address 2837 * pair. 2838 * In/out conditions: 2839 * The mapping must already exist in the pmap. 2840 */ 2841 2842 void 2843 pmap_unwire(struct pmap *pmap, vaddr_t va) 2844 { 2845 pt_entry_t *ptes; 2846 struct vm_page *pg; 2847 paddr_t pa; 2848 2849 PMAP_MAP_TO_HEAD_LOCK(); 2850 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2851 2852 if (pmap_pde_v(pmap_pde(pmap, va))) { 2853 #ifdef DIAGNOSTIC 2854 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 2855 panic("pmap_unwire: invalid L2 PTE"); 2856 #endif 2857 /* Extract the physical address of the page */ 2858 pa = l2pte_pa(ptes[arm_btop(va)]); 2859 2860 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 2861 goto out; 2862 2863 /* Update the wired bit in the pv entry for this page. */ 2864 simple_lock(&pg->mdpage.pvh_slock); 2865 (void) pmap_modify_pv(pmap, va, pg, PVF_WIRED, 0); 2866 simple_unlock(&pg->mdpage.pvh_slock); 2867 } 2868 #ifdef DIAGNOSTIC 2869 else { 2870 panic("pmap_unwire: invalid L1 PTE"); 2871 } 2872 #endif 2873 out: 2874 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2875 PMAP_MAP_TO_HEAD_UNLOCK(); 2876 } 2877 2878 /* 2879 * Routine: pmap_extract 2880 * Function: 2881 * Extract the physical page address associated 2882 * with the given map/virtual_address pair. 2883 */ 2884 boolean_t 2885 pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap) 2886 { 2887 pd_entry_t *pde; 2888 pt_entry_t *pte, *ptes; 2889 paddr_t pa; 2890 2891 PDEBUG(5, printf("pmap_extract: pmap=%p, va=0x%08lx -> ", pmap, va)); 2892 2893 ptes = pmap_map_ptes(pmap); /* locks pmap */ 2894 2895 pde = pmap_pde(pmap, va); 2896 pte = &ptes[arm_btop(va)]; 2897 2898 if (pmap_pde_section(pde)) { 2899 pa = (*pde & L1_S_FRAME) | (va & L1_S_OFFSET); 2900 PDEBUG(5, printf("section pa=0x%08lx\n", pa)); 2901 goto out; 2902 } else if (pmap_pde_page(pde) == 0 || pmap_pte_v(pte) == 0) { 2903 PDEBUG(5, printf("no mapping\n")); 2904 goto failed; 2905 } 2906 2907 if ((*pte & L2_TYPE_MASK) == L2_TYPE_L) { 2908 pa = (*pte & L2_L_FRAME) | (va & L2_L_OFFSET); 2909 PDEBUG(5, printf("large page pa=0x%08lx\n", pa)); 2910 goto out; 2911 } 2912 2913 pa = (*pte & L2_S_FRAME) | (va & L2_S_OFFSET); 2914 PDEBUG(5, printf("small page pa=0x%08lx\n", pa)); 2915 2916 out: 2917 if (pap != NULL) 2918 *pap = pa; 2919 2920 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2921 return (TRUE); 2922 2923 failed: 2924 pmap_unmap_ptes(pmap); /* unlocks pmap */ 2925 return (FALSE); 2926 } 2927 2928 2929 /* 2930 * pmap_copy: 2931 * 2932 * Copy the range specified by src_addr/len from the source map to the 2933 * range dst_addr/len in the destination map. 2934 * 2935 * This routine is only advisory and need not do anything. 2936 */ 2937 /* Call deleted in <arm/arm32/pmap.h> */ 2938 2939 #if defined(PMAP_DEBUG) 2940 void 2941 pmap_dump_pvlist(phys, m) 2942 vaddr_t phys; 2943 char *m; 2944 { 2945 struct vm_page *pg; 2946 struct pv_entry *pv; 2947 2948 if ((pg = PHYS_TO_VM_PAGE(phys)) == NULL) { 2949 printf("INVALID PA\n"); 2950 return; 2951 } 2952 simple_lock(&pg->mdpage.pvh_slock); 2953 printf("%s %08lx:", m, phys); 2954 if (pg->mdpage.pvh_list == NULL) { 2955 printf(" no mappings\n"); 2956 return; 2957 } 2958 2959 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) 2960 printf(" pmap %p va %08lx flags %08x", pv->pv_pmap, 2961 pv->pv_va, pv->pv_flags); 2962 2963 printf("\n"); 2964 simple_unlock(&pg->mdpage.pvh_slock); 2965 } 2966 2967 #endif /* PMAP_DEBUG */ 2968 2969 static pt_entry_t * 2970 pmap_map_ptes(struct pmap *pmap) 2971 { 2972 struct proc *p; 2973 2974 /* the kernel's pmap is always accessible */ 2975 if (pmap == pmap_kernel()) { 2976 return (pt_entry_t *)PTE_BASE; 2977 } 2978 2979 if (pmap_is_curpmap(pmap)) { 2980 simple_lock(&pmap->pm_obj.vmobjlock); 2981 return (pt_entry_t *)PTE_BASE; 2982 } 2983 2984 p = curproc; 2985 KDASSERT(p != NULL); 2986 2987 /* need to lock both curpmap and pmap: use ordered locking */ 2988 if ((vaddr_t) pmap < (vaddr_t) p->p_vmspace->vm_map.pmap) { 2989 simple_lock(&pmap->pm_obj.vmobjlock); 2990 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2991 } else { 2992 simple_lock(&p->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 2993 simple_lock(&pmap->pm_obj.vmobjlock); 2994 } 2995 2996 pmap_map_in_l1(p->p_vmspace->vm_map.pmap, APTE_BASE, pmap->pm_pptpt, 2997 FALSE); 2998 cpu_tlb_flushD(); 2999 cpu_cpwait(); 3000 return (pt_entry_t *)APTE_BASE; 3001 } 3002 3003 /* 3004 * pmap_unmap_ptes: unlock the PTE mapping of "pmap" 3005 */ 3006 3007 static void 3008 pmap_unmap_ptes(struct pmap *pmap) 3009 { 3010 3011 if (pmap == pmap_kernel()) { 3012 return; 3013 } 3014 if (pmap_is_curpmap(pmap)) { 3015 simple_unlock(&pmap->pm_obj.vmobjlock); 3016 } else { 3017 KDASSERT(curproc != NULL); 3018 simple_unlock(&pmap->pm_obj.vmobjlock); 3019 simple_unlock( 3020 &curproc->p_vmspace->vm_map.pmap->pm_obj.vmobjlock); 3021 } 3022 } 3023 3024 /* 3025 * Modify pte bits for all ptes corresponding to the given physical address. 3026 * We use `maskbits' rather than `clearbits' because we're always passing 3027 * constants and the latter would require an extra inversion at run-time. 3028 */ 3029 3030 static void 3031 pmap_clearbit(struct vm_page *pg, u_int maskbits) 3032 { 3033 struct pv_entry *pv; 3034 pt_entry_t *ptes; 3035 vaddr_t va; 3036 int tlbentry; 3037 3038 PDEBUG(1, printf("pmap_clearbit: pa=%08lx mask=%08x\n", 3039 VM_PAGE_TO_PHYS(pg), maskbits)); 3040 3041 tlbentry = 0; 3042 3043 PMAP_HEAD_TO_MAP_LOCK(); 3044 simple_lock(&pg->mdpage.pvh_slock); 3045 3046 /* 3047 * Clear saved attributes (modify, reference) 3048 */ 3049 pg->mdpage.pvh_attrs &= ~maskbits; 3050 3051 if (pg->mdpage.pvh_list == NULL) { 3052 simple_unlock(&pg->mdpage.pvh_slock); 3053 PMAP_HEAD_TO_MAP_UNLOCK(); 3054 return; 3055 } 3056 3057 /* 3058 * Loop over all current mappings setting/clearing as appropos 3059 */ 3060 for (pv = pg->mdpage.pvh_list; pv; pv = pv->pv_next) { 3061 va = pv->pv_va; 3062 pv->pv_flags &= ~maskbits; 3063 ptes = pmap_map_ptes(pv->pv_pmap); /* locks pmap */ 3064 KASSERT(pmap_pde_v(pmap_pde(pv->pv_pmap, va))); 3065 if (maskbits & (PVF_WRITE|PVF_MOD)) { 3066 if ((pv->pv_flags & PVF_NC)) { 3067 /* 3068 * Entry is not cacheable: reenable 3069 * the cache, nothing to flush 3070 * 3071 * Don't turn caching on again if this 3072 * is a modified emulation. This 3073 * would be inconsitent with the 3074 * settings created by 3075 * pmap_vac_me_harder(). 3076 * 3077 * There's no need to call 3078 * pmap_vac_me_harder() here: all 3079 * pages are loosing their write 3080 * permission. 3081 * 3082 */ 3083 if (maskbits & PVF_WRITE) { 3084 ptes[arm_btop(va)] |= 3085 pte_l2_s_cache_mode; 3086 pv->pv_flags &= ~PVF_NC; 3087 } 3088 } else if (pmap_is_curpmap(pv->pv_pmap)) { 3089 /* 3090 * Entry is cacheable: check if pmap is 3091 * current if it is flush it, 3092 * otherwise it won't be in the cache 3093 */ 3094 cpu_idcache_wbinv_range(pv->pv_va, NBPG); 3095 } 3096 3097 /* make the pte read only */ 3098 ptes[arm_btop(va)] &= ~L2_S_PROT_W; 3099 } 3100 3101 if (maskbits & PVF_REF) 3102 ptes[arm_btop(va)] = 3103 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_TYPE_INV; 3104 3105 if (pmap_is_curpmap(pv->pv_pmap)) { 3106 /* 3107 * if we had cacheable pte's we'd clean the 3108 * pte out to memory here 3109 * 3110 * flush tlb entry as it's in the current pmap 3111 */ 3112 cpu_tlb_flushID_SE(pv->pv_va); 3113 } 3114 pmap_unmap_ptes(pv->pv_pmap); /* unlocks pmap */ 3115 } 3116 cpu_cpwait(); 3117 3118 simple_unlock(&pg->mdpage.pvh_slock); 3119 PMAP_HEAD_TO_MAP_UNLOCK(); 3120 } 3121 3122 /* 3123 * pmap_clear_modify: 3124 * 3125 * Clear the "modified" attribute for a page. 3126 */ 3127 boolean_t 3128 pmap_clear_modify(struct vm_page *pg) 3129 { 3130 boolean_t rv; 3131 3132 if (pg->mdpage.pvh_attrs & PVF_MOD) { 3133 rv = TRUE; 3134 pmap_clearbit(pg, PVF_MOD); 3135 } else 3136 rv = FALSE; 3137 3138 PDEBUG(0, printf("pmap_clear_modify pa=%08lx -> %d\n", 3139 VM_PAGE_TO_PHYS(pg), rv)); 3140 3141 return (rv); 3142 } 3143 3144 /* 3145 * pmap_clear_reference: 3146 * 3147 * Clear the "referenced" attribute for a page. 3148 */ 3149 boolean_t 3150 pmap_clear_reference(struct vm_page *pg) 3151 { 3152 boolean_t rv; 3153 3154 if (pg->mdpage.pvh_attrs & PVF_REF) { 3155 rv = TRUE; 3156 pmap_clearbit(pg, PVF_REF); 3157 } else 3158 rv = FALSE; 3159 3160 PDEBUG(0, printf("pmap_clear_reference pa=%08lx -> %d\n", 3161 VM_PAGE_TO_PHYS(pg), rv)); 3162 3163 return (rv); 3164 } 3165 3166 /* 3167 * pmap_is_modified: 3168 * 3169 * Test if a page has the "modified" attribute. 3170 */ 3171 /* See <arm/arm32/pmap.h> */ 3172 3173 /* 3174 * pmap_is_referenced: 3175 * 3176 * Test if a page has the "referenced" attribute. 3177 */ 3178 /* See <arm/arm32/pmap.h> */ 3179 3180 int 3181 pmap_modified_emulation(struct pmap *pmap, vaddr_t va) 3182 { 3183 pt_entry_t *ptes; 3184 struct vm_page *pg; 3185 paddr_t pa; 3186 u_int flags; 3187 int rv = 0; 3188 3189 PDEBUG(2, printf("pmap_modified_emulation\n")); 3190 3191 PMAP_MAP_TO_HEAD_LOCK(); 3192 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3193 3194 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3195 PDEBUG(2, printf("L1 PTE invalid\n")); 3196 goto out; 3197 } 3198 3199 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3200 3201 /* Check for a invalid pte */ 3202 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3203 goto out; 3204 3205 /* This can happen if user code tries to access kernel memory. */ 3206 if ((ptes[arm_btop(va)] & L2_S_PROT_W) != 0) 3207 goto out; 3208 3209 /* Extract the physical address of the page */ 3210 pa = l2pte_pa(ptes[arm_btop(va)]); 3211 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3212 goto out; 3213 3214 /* Get the current flags for this page. */ 3215 simple_lock(&pg->mdpage.pvh_slock); 3216 3217 flags = pmap_modify_pv(pmap, va, pg, 0, 0); 3218 PDEBUG(2, printf("pmap_modified_emulation: flags = %08x\n", flags)); 3219 3220 /* 3221 * Do the flags say this page is writable ? If not then it is a 3222 * genuine write fault. If yes then the write fault is our fault 3223 * as we did not reflect the write access in the PTE. Now we know 3224 * a write has occurred we can correct this and also set the 3225 * modified bit 3226 */ 3227 if (~flags & PVF_WRITE) { 3228 simple_unlock(&pg->mdpage.pvh_slock); 3229 goto out; 3230 } 3231 3232 PDEBUG(0, 3233 printf("pmap_modified_emulation: Got a hit va=%08lx, pte = %08x\n", 3234 va, ptes[arm_btop(va)])); 3235 pg->mdpage.pvh_attrs |= PVF_REF | PVF_MOD; 3236 3237 /* 3238 * Re-enable write permissions for the page. No need to call 3239 * pmap_vac_me_harder(), since this is just a 3240 * modified-emulation fault, and the PVF_WRITE bit isn't changing. 3241 * We've already set the cacheable bits based on the assumption 3242 * that we can write to this page. 3243 */ 3244 ptes[arm_btop(va)] = 3245 (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; 3246 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3247 3248 simple_unlock(&pg->mdpage.pvh_slock); 3249 3250 cpu_tlb_flushID_SE(va); 3251 cpu_cpwait(); 3252 rv = 1; 3253 out: 3254 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3255 PMAP_MAP_TO_HEAD_UNLOCK(); 3256 return (rv); 3257 } 3258 3259 int 3260 pmap_handled_emulation(struct pmap *pmap, vaddr_t va) 3261 { 3262 pt_entry_t *ptes; 3263 struct vm_page *pg; 3264 paddr_t pa; 3265 int rv = 0; 3266 3267 PDEBUG(2, printf("pmap_handled_emulation\n")); 3268 3269 PMAP_MAP_TO_HEAD_LOCK(); 3270 ptes = pmap_map_ptes(pmap); /* locks pmap */ 3271 3272 if (pmap_pde_v(pmap_pde(pmap, va)) == 0) { 3273 PDEBUG(2, printf("L1 PTE invalid\n")); 3274 goto out; 3275 } 3276 3277 PDEBUG(1, printf("pte=%08x\n", ptes[arm_btop(va)])); 3278 3279 /* Check for invalid pte */ 3280 if (l2pte_valid(ptes[arm_btop(va)]) == 0) 3281 goto out; 3282 3283 /* This can happen if user code tries to access kernel memory. */ 3284 if ((ptes[arm_btop(va)] & L2_TYPE_MASK) != L2_TYPE_INV) 3285 goto out; 3286 3287 /* Extract the physical address of the page */ 3288 pa = l2pte_pa(ptes[arm_btop(va)]); 3289 if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) 3290 goto out; 3291 3292 simple_lock(&pg->mdpage.pvh_slock); 3293 3294 /* 3295 * Ok we just enable the pte and mark the attibs as handled 3296 * XXX Should we traverse the PV list and enable all PTEs? 3297 */ 3298 PDEBUG(0, 3299 printf("pmap_handled_emulation: Got a hit va=%08lx pte = %08x\n", 3300 va, ptes[arm_btop(va)])); 3301 pg->mdpage.pvh_attrs |= PVF_REF; 3302 3303 ptes[arm_btop(va)] = (ptes[arm_btop(va)] & ~L2_TYPE_MASK) | L2_S_PROTO; 3304 PDEBUG(0, printf("->(%08x)\n", ptes[arm_btop(va)])); 3305 3306 simple_unlock(&pg->mdpage.pvh_slock); 3307 3308 cpu_tlb_flushID_SE(va); 3309 cpu_cpwait(); 3310 rv = 1; 3311 out: 3312 pmap_unmap_ptes(pmap); /* unlocks pmap */ 3313 PMAP_MAP_TO_HEAD_UNLOCK(); 3314 return (rv); 3315 } 3316 3317 /* 3318 * pmap_collect: free resources held by a pmap 3319 * 3320 * => optional function. 3321 * => called when a process is swapped out to free memory. 3322 */ 3323 3324 void 3325 pmap_collect(struct pmap *pmap) 3326 { 3327 } 3328 3329 /* 3330 * Routine: pmap_procwr 3331 * 3332 * Function: 3333 * Synchronize caches corresponding to [addr, addr+len) in p. 3334 * 3335 */ 3336 void 3337 pmap_procwr(struct proc *p, vaddr_t va, int len) 3338 { 3339 /* We only need to do anything if it is the current process. */ 3340 if (p == curproc) 3341 cpu_icache_sync_range(va, len); 3342 } 3343 /* 3344 * PTP functions 3345 */ 3346 3347 /* 3348 * pmap_get_ptp: get a PTP (if there isn't one, allocate a new one) 3349 * 3350 * => pmap should NOT be pmap_kernel() 3351 * => pmap should be locked 3352 */ 3353 3354 static struct vm_page * 3355 pmap_get_ptp(struct pmap *pmap, vaddr_t va) 3356 { 3357 struct vm_page *ptp; 3358 3359 if (pmap_pde_page(pmap_pde(pmap, va))) { 3360 3361 /* valid... check hint (saves us a PA->PG lookup) */ 3362 if (pmap->pm_ptphint && 3363 (pmap->pm_pdir[pmap_pdei(va)] & L2_S_FRAME) == 3364 VM_PAGE_TO_PHYS(pmap->pm_ptphint)) 3365 return (pmap->pm_ptphint); 3366 ptp = uvm_pagelookup(&pmap->pm_obj, va); 3367 #ifdef DIAGNOSTIC 3368 if (ptp == NULL) 3369 panic("pmap_get_ptp: unmanaged user PTP"); 3370 #endif 3371 pmap->pm_ptphint = ptp; 3372 return(ptp); 3373 } 3374 3375 /* allocate a new PTP (updates ptphint) */ 3376 return(pmap_alloc_ptp(pmap, va)); 3377 } 3378 3379 /* 3380 * pmap_alloc_ptp: allocate a PTP for a PMAP 3381 * 3382 * => pmap should already be locked by caller 3383 * => we use the ptp's wire_count to count the number of active mappings 3384 * in the PTP (we start it at one to prevent any chance this PTP 3385 * will ever leak onto the active/inactive queues) 3386 */ 3387 3388 /*__inline */ static struct vm_page * 3389 pmap_alloc_ptp(struct pmap *pmap, vaddr_t va) 3390 { 3391 struct vm_page *ptp; 3392 3393 ptp = uvm_pagealloc(&pmap->pm_obj, va, NULL, 3394 UVM_PGA_USERESERVE|UVM_PGA_ZERO); 3395 if (ptp == NULL) 3396 return (NULL); 3397 3398 /* got one! */ 3399 ptp->flags &= ~PG_BUSY; /* never busy */ 3400 ptp->wire_count = 1; /* no mappings yet */ 3401 pmap_map_in_l1(pmap, va, VM_PAGE_TO_PHYS(ptp), TRUE); 3402 pmap->pm_stats.resident_count++; /* count PTP as resident */ 3403 pmap->pm_ptphint = ptp; 3404 return (ptp); 3405 } 3406 3407 vaddr_t 3408 pmap_growkernel(vaddr_t maxkvaddr) 3409 { 3410 struct pmap *kpm = pmap_kernel(), *pm; 3411 int s; 3412 paddr_t ptaddr; 3413 struct vm_page *ptp; 3414 3415 if (maxkvaddr <= pmap_curmaxkvaddr) 3416 goto out; /* we are OK */ 3417 NPDEBUG(PDB_GROWKERN, printf("pmap_growkernel: growing kernel from %lx to %lx\n", 3418 pmap_curmaxkvaddr, maxkvaddr)); 3419 3420 /* 3421 * whoops! we need to add kernel PTPs 3422 */ 3423 3424 s = splhigh(); /* to be safe */ 3425 simple_lock(&kpm->pm_obj.vmobjlock); 3426 /* due to the way the arm pmap works we map 4MB at a time */ 3427 for (/*null*/ ; pmap_curmaxkvaddr < maxkvaddr; 3428 pmap_curmaxkvaddr += 4 * L1_S_SIZE) { 3429 3430 if (uvm.page_init_done == FALSE) { 3431 3432 /* 3433 * we're growing the kernel pmap early (from 3434 * uvm_pageboot_alloc()). this case must be 3435 * handled a little differently. 3436 */ 3437 3438 if (uvm_page_physget(&ptaddr) == FALSE) 3439 panic("pmap_growkernel: out of memory"); 3440 pmap_zero_page(ptaddr); 3441 3442 /* map this page in */ 3443 pmap_map_in_l1(kpm, pmap_curmaxkvaddr, ptaddr, TRUE); 3444 3445 /* count PTP as resident */ 3446 kpm->pm_stats.resident_count++; 3447 continue; 3448 } 3449 3450 /* 3451 * THIS *MUST* BE CODED SO AS TO WORK IN THE 3452 * pmap_initialized == FALSE CASE! WE MAY BE 3453 * INVOKED WHILE pmap_init() IS RUNNING! 3454 */ 3455 3456 if ((ptp = pmap_alloc_ptp(kpm, pmap_curmaxkvaddr)) == NULL) 3457 panic("pmap_growkernel: alloc ptp failed"); 3458 3459 /* distribute new kernel PTP to all active pmaps */ 3460 simple_lock(&pmaps_lock); 3461 LIST_FOREACH(pm, &pmaps, pm_list) { 3462 pmap_map_in_l1(pm, pmap_curmaxkvaddr, 3463 VM_PAGE_TO_PHYS(ptp), TRUE); 3464 } 3465 3466 simple_unlock(&pmaps_lock); 3467 } 3468 3469 /* 3470 * flush out the cache, expensive but growkernel will happen so 3471 * rarely 3472 */ 3473 cpu_tlb_flushD(); 3474 cpu_cpwait(); 3475 3476 simple_unlock(&kpm->pm_obj.vmobjlock); 3477 splx(s); 3478 3479 out: 3480 return (pmap_curmaxkvaddr); 3481 } 3482 3483 /************************ Utility routines ****************************/ 3484 3485 /* 3486 * vector_page_setprot: 3487 * 3488 * Manipulate the protection of the vector page. 3489 */ 3490 void 3491 vector_page_setprot(int prot) 3492 { 3493 pt_entry_t *pte; 3494 3495 pte = vtopte(vector_page); 3496 3497 *pte = (*pte & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); 3498 cpu_tlb_flushD_SE(vector_page); 3499 cpu_cpwait(); 3500 } 3501 3502 /************************ Bootstrapping routines ****************************/ 3503 3504 /* 3505 * This list exists for the benefit of pmap_map_chunk(). It keeps track 3506 * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can 3507 * find them as necessary. 3508 * 3509 * Note that the data on this list is not valid after initarm() returns. 3510 */ 3511 SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); 3512 3513 static vaddr_t 3514 kernel_pt_lookup(paddr_t pa) 3515 { 3516 pv_addr_t *pv; 3517 3518 SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { 3519 if (pv->pv_pa == pa) 3520 return (pv->pv_va); 3521 } 3522 return (0); 3523 } 3524 3525 /* 3526 * pmap_map_section: 3527 * 3528 * Create a single section mapping. 3529 */ 3530 void 3531 pmap_map_section(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3532 { 3533 pd_entry_t *pde = (pd_entry_t *) l1pt; 3534 pd_entry_t fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3535 3536 KASSERT(((va | pa) & L1_S_OFFSET) == 0); 3537 3538 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3539 L1_S_PROT(PTE_KERNEL, prot) | fl; 3540 } 3541 3542 /* 3543 * pmap_map_entry: 3544 * 3545 * Create a single page mapping. 3546 */ 3547 void 3548 pmap_map_entry(vaddr_t l1pt, vaddr_t va, paddr_t pa, int prot, int cache) 3549 { 3550 pd_entry_t *pde = (pd_entry_t *) l1pt; 3551 pt_entry_t fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3552 pt_entry_t *pte; 3553 3554 KASSERT(((va | pa) & PGOFSET) == 0); 3555 3556 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3557 panic("pmap_map_entry: no L2 table for VA 0x%08lx", va); 3558 3559 pte = (pt_entry_t *) 3560 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3561 if (pte == NULL) 3562 panic("pmap_map_entry: can't find L2 table for VA 0x%08lx", va); 3563 3564 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3565 L2_S_PROT(PTE_KERNEL, prot) | fl; 3566 } 3567 3568 /* 3569 * pmap_link_l2pt: 3570 * 3571 * Link the L2 page table specified by "pa" into the L1 3572 * page table at the slot for "va". 3573 */ 3574 void 3575 pmap_link_l2pt(vaddr_t l1pt, vaddr_t va, pv_addr_t *l2pv) 3576 { 3577 pd_entry_t *pde = (pd_entry_t *) l1pt; 3578 u_int slot = va >> L1_S_SHIFT; 3579 3580 KASSERT((l2pv->pv_pa & PGOFSET) == 0); 3581 3582 pde[slot + 0] = L1_C_PROTO | (l2pv->pv_pa + 0x000); 3583 pde[slot + 1] = L1_C_PROTO | (l2pv->pv_pa + 0x400); 3584 pde[slot + 2] = L1_C_PROTO | (l2pv->pv_pa + 0x800); 3585 pde[slot + 3] = L1_C_PROTO | (l2pv->pv_pa + 0xc00); 3586 3587 SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); 3588 } 3589 3590 /* 3591 * pmap_map_chunk: 3592 * 3593 * Map a chunk of memory using the most efficient mappings 3594 * possible (section, large page, small page) into the 3595 * provided L1 and L2 tables at the specified virtual address. 3596 */ 3597 vsize_t 3598 pmap_map_chunk(vaddr_t l1pt, vaddr_t va, paddr_t pa, vsize_t size, 3599 int prot, int cache) 3600 { 3601 pd_entry_t *pde = (pd_entry_t *) l1pt; 3602 pt_entry_t *pte, fl; 3603 vsize_t resid; 3604 int i; 3605 3606 resid = (size + (NBPG - 1)) & ~(NBPG - 1); 3607 3608 if (l1pt == 0) 3609 panic("pmap_map_chunk: no L1 table provided"); 3610 3611 #ifdef VERBOSE_INIT_ARM 3612 printf("pmap_map_chunk: pa=0x%lx va=0x%lx size=0x%lx resid=0x%lx " 3613 "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); 3614 #endif 3615 3616 size = resid; 3617 3618 while (resid > 0) { 3619 /* See if we can use a section mapping. */ 3620 if (((pa | va) & L1_S_OFFSET) == 0 && 3621 resid >= L1_S_SIZE) { 3622 fl = (cache == PTE_CACHE) ? pte_l1_s_cache_mode : 0; 3623 #ifdef VERBOSE_INIT_ARM 3624 printf("S"); 3625 #endif 3626 pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | 3627 L1_S_PROT(PTE_KERNEL, prot) | fl; 3628 va += L1_S_SIZE; 3629 pa += L1_S_SIZE; 3630 resid -= L1_S_SIZE; 3631 continue; 3632 } 3633 3634 /* 3635 * Ok, we're going to use an L2 table. Make sure 3636 * one is actually in the corresponding L1 slot 3637 * for the current VA. 3638 */ 3639 if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) 3640 panic("pmap_map_chunk: no L2 table for VA 0x%08lx", va); 3641 3642 pte = (pt_entry_t *) 3643 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3644 if (pte == NULL) 3645 panic("pmap_map_chunk: can't find L2 table for VA" 3646 "0x%08lx", va); 3647 3648 /* See if we can use a L2 large page mapping. */ 3649 if (((pa | va) & L2_L_OFFSET) == 0 && 3650 resid >= L2_L_SIZE) { 3651 fl = (cache == PTE_CACHE) ? pte_l2_l_cache_mode : 0; 3652 #ifdef VERBOSE_INIT_ARM 3653 printf("L"); 3654 #endif 3655 for (i = 0; i < 16; i++) { 3656 pte[((va >> PGSHIFT) & 0x3f0) + i] = 3657 L2_L_PROTO | pa | 3658 L2_L_PROT(PTE_KERNEL, prot) | fl; 3659 } 3660 va += L2_L_SIZE; 3661 pa += L2_L_SIZE; 3662 resid -= L2_L_SIZE; 3663 continue; 3664 } 3665 3666 /* Use a small page mapping. */ 3667 fl = (cache == PTE_CACHE) ? pte_l2_s_cache_mode : 0; 3668 #ifdef VERBOSE_INIT_ARM 3669 printf("P"); 3670 #endif 3671 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3672 L2_S_PROT(PTE_KERNEL, prot) | fl; 3673 va += NBPG; 3674 pa += NBPG; 3675 resid -= NBPG; 3676 } 3677 #ifdef VERBOSE_INIT_ARM 3678 printf("\n"); 3679 #endif 3680 return (size); 3681 } 3682 3683 /********************** PTE initialization routines **************************/ 3684 3685 /* 3686 * These routines are called when the CPU type is identified to set up 3687 * the PTE prototypes, cache modes, etc. 3688 * 3689 * The variables are always here, just in case LKMs need to reference 3690 * them (though, they shouldn't). 3691 */ 3692 3693 pt_entry_t pte_l1_s_cache_mode; 3694 pt_entry_t pte_l1_s_cache_mask; 3695 3696 pt_entry_t pte_l2_l_cache_mode; 3697 pt_entry_t pte_l2_l_cache_mask; 3698 3699 pt_entry_t pte_l2_s_cache_mode; 3700 pt_entry_t pte_l2_s_cache_mask; 3701 3702 pt_entry_t pte_l2_s_prot_u; 3703 pt_entry_t pte_l2_s_prot_w; 3704 pt_entry_t pte_l2_s_prot_mask; 3705 3706 pt_entry_t pte_l1_s_proto; 3707 pt_entry_t pte_l1_c_proto; 3708 pt_entry_t pte_l2_s_proto; 3709 3710 void (*pmap_copy_page_func)(paddr_t, paddr_t); 3711 void (*pmap_zero_page_func)(paddr_t); 3712 3713 #if ARM_MMU_GENERIC == 1 3714 void 3715 pmap_pte_init_generic(void) 3716 { 3717 3718 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3719 pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; 3720 3721 pte_l2_l_cache_mode = L2_B|L2_C; 3722 pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; 3723 3724 pte_l2_s_cache_mode = L2_B|L2_C; 3725 pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; 3726 3727 pte_l2_s_prot_u = L2_S_PROT_U_generic; 3728 pte_l2_s_prot_w = L2_S_PROT_W_generic; 3729 pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; 3730 3731 pte_l1_s_proto = L1_S_PROTO_generic; 3732 pte_l1_c_proto = L1_C_PROTO_generic; 3733 pte_l2_s_proto = L2_S_PROTO_generic; 3734 3735 pmap_copy_page_func = pmap_copy_page_generic; 3736 pmap_zero_page_func = pmap_zero_page_generic; 3737 } 3738 3739 #if defined(CPU_ARM9) 3740 void 3741 pmap_pte_init_arm9(void) 3742 { 3743 3744 /* 3745 * ARM9 is compatible with generic, but we want to use 3746 * write-through caching for now. 3747 */ 3748 pmap_pte_init_generic(); 3749 3750 pte_l1_s_cache_mode = L1_S_C; 3751 pte_l2_l_cache_mode = L2_C; 3752 pte_l2_s_cache_mode = L2_C; 3753 } 3754 #endif /* CPU_ARM9 */ 3755 #endif /* ARM_MMU_GENERIC == 1 */ 3756 3757 #if ARM_MMU_XSCALE == 1 3758 void 3759 pmap_pte_init_xscale(void) 3760 { 3761 3762 pte_l1_s_cache_mode = L1_S_B|L1_S_C; 3763 pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; 3764 3765 pte_l2_l_cache_mode = L2_B|L2_C; 3766 pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; 3767 3768 pte_l2_s_cache_mode = L2_B|L2_C; 3769 pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; 3770 3771 pte_l2_s_prot_u = L2_S_PROT_U_xscale; 3772 pte_l2_s_prot_w = L2_S_PROT_W_xscale; 3773 pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; 3774 3775 pte_l1_s_proto = L1_S_PROTO_xscale; 3776 pte_l1_c_proto = L1_C_PROTO_xscale; 3777 pte_l2_s_proto = L2_S_PROTO_xscale; 3778 3779 pmap_copy_page_func = pmap_copy_page_xscale; 3780 pmap_zero_page_func = pmap_zero_page_xscale; 3781 } 3782 3783 #if defined(CPU_XSCALE_80200) 3784 void 3785 pmap_pte_init_i80200(void) 3786 { 3787 3788 /* 3789 * Use write-through caching on the i80200. 3790 */ 3791 pmap_pte_init_xscale(); 3792 pte_l1_s_cache_mode = L1_S_C; 3793 pte_l2_l_cache_mode = L2_C; 3794 pte_l2_s_cache_mode = L2_C; 3795 } 3796 #endif /* CPU_XSCALE_80200 */ 3797 3798 /* 3799 * xscale_setup_minidata: 3800 * 3801 * Set up the mini-data cache clean area. We require the 3802 * caller to allocate the right amount of physically and 3803 * virtually contiguous space. 3804 */ 3805 void 3806 xscale_setup_minidata(vaddr_t l1pt, vaddr_t va, paddr_t pa) 3807 { 3808 extern vaddr_t xscale_minidata_clean_addr; 3809 extern vsize_t xscale_minidata_clean_size; /* already initialized */ 3810 pd_entry_t *pde = (pd_entry_t *) l1pt; 3811 pt_entry_t *pte; 3812 vsize_t size; 3813 3814 xscale_minidata_clean_addr = va; 3815 3816 /* Round it to page size. */ 3817 size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; 3818 3819 for (; size != 0; 3820 va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { 3821 pte = (pt_entry_t *) 3822 kernel_pt_lookup(pde[va >> L1_S_SHIFT] & L2_S_FRAME); 3823 if (pte == NULL) 3824 panic("xscale_setup_minidata: can't find L2 table for " 3825 "VA 0x%08lx", va); 3826 pte[(va >> PGSHIFT) & 0x3ff] = L2_S_PROTO | pa | 3827 L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | 3828 L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); 3829 } 3830 } 3831 #endif /* ARM_MMU_XSCALE == 1 */ 3832
Indexes created Mon Sep 29 18:09:42 GMT 2025