1 /* $NetBSD: pmap.c,v 1.44 2024/10/31 10:09:56 andvar Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 1999, 2000, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /*- 34 * Copyright (c) 1991 Regents of the University of California. 35 * All rights reserved. 36 * Copyright (c) 1994 John S. Dyson 37 * All rights reserved. 38 * Copyright (c) 1994 David Greenman 39 * All rights reserved. 40 * Copyright (c) 1998,2000 Doug Rabson 41 * All rights reserved. 42 * 43 * This code is derived from software contributed to Berkeley by 44 * the Systems Programming Group of the University of Utah Computer 45 * Science Department and William Jolitz of UUNET Technologies Inc. 46 * 47 * Redistribution and use in source and binary forms, with or without 48 * modification, are permitted provided that the following conditions 49 * are met: 50 * 1. Redistributions of source code must retain the above copyright 51 * notice, this list of conditions and the following disclaimer. 52 * 2. Redistributions in binary form must reproduce the above copyright 53 * notice, this list of conditions and the following disclaimer in the 54 * documentation and/or other materials provided with the distribution. 55 * 3. All advertising materials mentioning features or use of this software 56 * must display the following acknowledgement: 57 * This product includes software developed by the University of 58 * California, Berkeley and its contributors. 59 * 4. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 76 * from: i386 Id: pmap.c,v 1.193 1998/04/19 15:22:48 bde Exp 77 * with some ideas from NetBSD's alpha pmap 78 */ 79 80 /* __FBSDID("$FreeBSD: src/sys/ia64/ia64/pmap.c,v 1.172 2005/11/20 06:09:48 alc Exp $"); */ 81 82 #include <sys/cdefs.h> 83 84 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.44 2024/10/31 10:09:56 andvar Exp $"); 85 86 #include <sys/param.h> 87 #include <sys/atomic.h> 88 #include <sys/systm.h> 89 #include <sys/buf.h> 90 #include <sys/reboot.h> 91 #include <sys/lock.h> 92 #include <sys/pool.h> 93 #include <sys/sched.h> 94 #include <sys/bitops.h> 95 96 #include <uvm/uvm.h> 97 #include <uvm/uvm_physseg.h> 98 99 #include <machine/pal.h> 100 #include <machine/atomic.h> 101 #include <machine/pte.h> 102 #include <machine/cpufunc.h> 103 #include <machine/md_var.h> 104 #include <machine/vmparam.h> 105 106 /* 107 * Manages physical address maps. 108 * 109 * Since the information managed by this module is 110 * also stored by the logical address mapping module, 111 * this module may throw away valid virtual-to-physical 112 * mappings at almost any time. However, invalidations 113 * of virtual-to-physical mappings must be done as 114 * requested. 115 * 116 * In order to cope with hardware architectures which 117 * make virtual-to-physical map invalidates expensive, 118 * this module may delay invalidate or reduced protection 119 * operations until such time as they are actually 120 * necessary. This module is given full information as 121 * to which processors are currently using which maps, 122 * and to when physical maps must be made correct. 123 */ 124 125 /* 126 * Following the Linux model, region IDs are allocated in groups of 127 * eight so that a single region ID can be used for as many RRs as we 128 * want by encoding the RR number into the low bits of the ID. 129 * 130 * We reserve region ID 0 for the kernel and allocate the remaining 131 * IDs for user pmaps. 132 * 133 * Region 0-3: User virtually mapped 134 * Region 4: PBVM and special mappings 135 * Region 5: Kernel virtual memory 136 * Region 6: Direct-mapped uncacheable 137 * Region 7: Direct-mapped cacheable 138 */ 139 140 #if !defined(DIAGNOSTIC) 141 #define PMAP_INLINE __inline 142 #else 143 #define PMAP_INLINE 144 #endif 145 146 #ifdef PV_STATS 147 #define PV_STAT(x) do { x ; } while (0) 148 #else 149 #define PV_STAT(x) do { } while (0) 150 #endif 151 152 #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED) 153 #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY) 154 #define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX) 155 #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED) 156 #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK) 157 #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT) 158 #define pmap_prot(lpte) (((lpte)->pte & PTE_PROT_MASK) >> 56) 159 #define pmap_wired(lpte) ((lpte)->pte & PTE_WIRED) 160 161 #define pmap_clear_accessed(lpte) (lpte)->pte &= ~PTE_ACCESSED 162 #define pmap_clear_dirty(lpte) (lpte)->pte &= ~PTE_DIRTY 163 #define pmap_clear_present(lpte) (lpte)->pte &= ~PTE_PRESENT 164 #define pmap_clear_wired(lpte) (lpte)->pte &= ~PTE_WIRED 165 166 #define pmap_set_wired(lpte) (lpte)->pte |= PTE_WIRED 167 168 /* 169 * Individual PV entries are stored in per-pmap chunks. This saves 170 * space by eliminating the need to record the pmap within every PV 171 * entry. 172 */ 173 #if PAGE_SIZE == 8192 174 #define _NPCM 6 175 #define _NPCPV 337 176 #define _NPCS 2 177 #elif PAGE_SIZE == 16384 178 #define _NPCM 11 179 #define _NPCPV 677 180 #define _NPCS 1 181 #else 182 #error "invalid page size" 183 #endif 184 185 struct pv_chunk { 186 pmap_t pc_pmap; 187 TAILQ_ENTRY(pv_chunk) pc_list; 188 u_long pc_map[_NPCM]; /* bitmap; 1 = free */ 189 TAILQ_ENTRY(pv_chunk) pc_lru; 190 u_long pc_spare[_NPCS]; 191 struct pv_entry pc_pventry[_NPCPV]; 192 }; 193 194 /* 195 * The VHPT bucket head structure. 196 */ 197 struct ia64_bucket { 198 uint64_t chain; 199 kmutex_t mutex; 200 u_int length; 201 }; 202 203 /* 204 * Statically allocated kernel pmap 205 */ 206 static struct pmap kernel_pmap_store;/* the kernel's pmap (proc0) */ 207 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; 208 209 vaddr_t virtual_avail; /* VA of first avail page (after kernel bss) */ 210 vaddr_t virtual_end; /* VA of last avail page (end of kernel AS) */ 211 212 /* XXX freebsd, needs to be sorted out */ 213 #define kernel_pmap pmap_kernel() 214 #define critical_enter() kpreempt_disable() 215 #define critical_exit() kpreempt_enable() 216 /* flags the entire page as dirty */ 217 #define vm_page_dirty(page) (page->flags &= ~PG_CLEAN) 218 #define vm_page_is_managed(page) (pmap_initialized && uvm_pageismanaged(VM_PAGE_TO_PHYS(page))) 219 220 /* 221 * Kernel virtual memory management. 222 */ 223 static int nkpt; 224 225 extern struct ia64_lpte ***ia64_kptdir; 226 227 #define KPTE_DIR0_INDEX(va) \ 228 (((va) >> (3*PAGE_SHIFT-8)) & ((1<<(PAGE_SHIFT-3))-1)) 229 #define KPTE_DIR1_INDEX(va) \ 230 (((va) >> (2*PAGE_SHIFT-5)) & ((1<<(PAGE_SHIFT-3))-1)) 231 #define KPTE_PTE_INDEX(va) \ 232 (((va) >> PAGE_SHIFT) & ((1<<(PAGE_SHIFT-5))-1)) 233 234 #define NKPTEPG (PAGE_SIZE / sizeof(struct ia64_lpte)) 235 236 vaddr_t kernel_vm_end; 237 238 /* Defaults for ptc.e. */ 239 /* 240 static uint64_t pmap_ptc_e_base = 0; 241 static uint32_t pmap_ptc_e_count1 = 1; 242 static uint32_t pmap_ptc_e_count2 = 1; 243 static uint32_t pmap_ptc_e_stride1 = 0; 244 static uint32_t pmap_ptc_e_stride2 = 0; 245 */ 246 /* Values for ptc.e. XXX values for SKI, add SKI kernel option methinks */ 247 static uint64_t pmap_ptc_e_base = 0x100000000; 248 static uint64_t pmap_ptc_e_count1 = 3; 249 static uint64_t pmap_ptc_e_count2 = 2; 250 static uint64_t pmap_ptc_e_stride1 = 0x2000; 251 static uint64_t pmap_ptc_e_stride2 = 0x100000000; 252 253 kmutex_t pmap_ptc_mutex; 254 255 /* 256 * Data for the RID allocator 257 */ 258 static int pmap_ridcount; 259 static int pmap_rididx; 260 static int pmap_ridmapsz; 261 static int pmap_ridmax; 262 static uint64_t *pmap_ridmap; 263 kmutex_t pmap_ridmutex; 264 265 static krwlock_t pvh_global_lock __attribute__ ((aligned (128))); 266 267 static pool_cache_t pmap_pool_cache; 268 269 /* 270 * Data for the pv entry allocation mechanism 271 */ 272 static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); 273 static int pv_entry_count; 274 275 /* 276 * Data for allocating PTEs for user processes. 277 */ 278 static pool_cache_t pte_pool_cache; 279 280 struct ia64_bucket *pmap_vhpt_bucket; 281 282 /* XXX For freebsd, these are sysctl variables */ 283 static uint64_t pmap_vhpt_nbuckets = 0; 284 uint64_t pmap_vhpt_log2size = 0; 285 static uint64_t pmap_vhpt_inserts = 0; 286 287 static bool pmap_initialized = false; /* Has pmap_init completed? */ 288 static uint64_t pmap_pages_stolen = 0; /* instrumentation */ 289 290 static struct ia64_lpte *pmap_find_vhpt(vaddr_t va); 291 292 static void free_pv_chunk(struct pv_chunk *pc); 293 static void free_pv_entry(pmap_t pmap, pv_entry_t pv); 294 static pv_entry_t get_pv_entry(pmap_t pmap, bool try); 295 static struct vm_page *pmap_pv_reclaim(pmap_t locked_pmap); 296 297 static void pmap_free_pte(struct ia64_lpte *pte, vaddr_t va); 298 static int pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, 299 vaddr_t va, pv_entry_t pv, int freepte); 300 static int pmap_remove_vhpt(vaddr_t va); 301 302 static vaddr_t pmap_steal_vhpt_memory(vsize_t); 303 304 static struct vm_page *vm_page_alloc1(void); 305 static void vm_page_free1(struct vm_page *pg); 306 307 static vm_memattr_t pmap_flags_to_memattr(u_int flags); 308 309 #if DEBUG 310 static void pmap_testout(void); 311 #endif 312 313 static void 314 pmap_initialize_vhpt(vaddr_t vhpt) 315 { 316 struct ia64_lpte *pte; 317 u_int i; 318 319 pte = (struct ia64_lpte *)vhpt; 320 for (i = 0; i < pmap_vhpt_nbuckets; i++) { 321 pte[i].pte = 0; 322 pte[i].itir = 0; 323 pte[i].tag = 1UL << 63; /* Invalid tag */ 324 pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i); 325 } 326 } 327 328 #ifdef MULTIPROCESSOR 329 vaddr_t 330 pmap_alloc_vhpt(void) 331 { 332 vaddr_t vhpt; 333 struct vm_page *m; 334 vsize_t size; 335 336 size = 1UL << pmap_vhpt_log2size; 337 m = vm_page_alloc_contig(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | 338 VM_ALLOC_WIRED, atop(size), 0UL, ~0UL, size, 0UL, 339 VM_MEMATTR_DEFAULT); 340 if (m != NULL) { 341 vhpt = IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 342 pmap_initialize_vhpt(vhpt); 343 return (vhpt); 344 } 345 return (0); 346 } 347 #endif 348 349 /* 350 * Bootstrap the system enough to run with virtual memory. 351 */ 352 void 353 pmap_bootstrap(void) 354 { 355 struct ia64_pal_result res; 356 vaddr_t base; 357 size_t size; 358 vsize_t bufsz; 359 int i, ridbits; 360 361 /* 362 * Query the PAL Code to find the loop parameters for the 363 * ptc.e instruction. 364 */ 365 res = ia64_call_pal_static(PAL_PTCE_INFO, 0, 0, 0); 366 if (res.pal_status != 0) 367 panic("Can't configure ptc.e parameters"); 368 pmap_ptc_e_base = res.pal_result[0]; 369 pmap_ptc_e_count1 = res.pal_result[1] >> 32; 370 pmap_ptc_e_count2 = res.pal_result[1] & ((1L<<32) - 1); 371 pmap_ptc_e_stride1 = res.pal_result[2] >> 32; 372 pmap_ptc_e_stride2 = res.pal_result[2] & ((1L<<32) - 1); 373 if (bootverbose) 374 printf("ptc.e base=0x%lx, count1=%ld, count2=%ld, " 375 "stride1=0x%lx, stride2=0x%lx\n", 376 pmap_ptc_e_base, 377 pmap_ptc_e_count1, 378 pmap_ptc_e_count2, 379 pmap_ptc_e_stride1, 380 pmap_ptc_e_stride2); 381 382 mutex_init(&pmap_ptc_mutex, MUTEX_DEFAULT, IPL_VM); 383 384 /* 385 * Setup RIDs. RIDs 0..7 are reserved for the kernel. 386 * 387 * We currently need at least 19 bits in the RID because PID_MAX 388 * can only be encoded in 17 bits and we need RIDs for 4 regions 389 * per process. With PID_MAX equalling 99999 this means that we 390 * need to be able to encode 399996 (=4*PID_MAX). 391 * The Itanium processor only has 18 bits and the architected 392 * minimum is exactly that. So, we cannot use a PID based scheme 393 * in those cases. Enter pmap_ridmap... 394 * We should avoid the map when running on a processor that has 395 * implemented enough bits. This means that we should pass the 396 * process/thread ID to pmap. This we currently don't do, so we 397 * use the map anyway. However, we don't want to allocate a map 398 * that is large enough to cover the range dictated by the number 399 * of bits in the RID, because that may result in a RID map of 400 * 2MB in size for a 24-bit RID. A 64KB map is enough. 401 * The bottomline: we create a 32KB map when the processor only 402 * implements 18 bits (or when we can't figure it out). Otherwise 403 * we create a 64KB map. 404 */ 405 res = ia64_call_pal_static(PAL_VM_SUMMARY, 0, 0, 0); 406 if (res.pal_status != 0) { 407 if (bootverbose) 408 printf("Can't read VM Summary - assuming 18 Region ID bits\n"); 409 ridbits = 18; /* guaranteed minimum */ 410 } else { 411 ridbits = (res.pal_result[1] >> 8) & 0xff; 412 if (bootverbose) 413 printf("Processor supports %d Region ID bits\n", 414 ridbits); 415 } 416 if (ridbits > 19) 417 ridbits = 19; 418 419 pmap_ridmax = (1 << ridbits); 420 pmap_ridmapsz = pmap_ridmax / 64; 421 pmap_ridmap = (uint64_t *)uvm_pageboot_alloc(pmap_ridmax / 8); 422 pmap_ridmap[0] |= 0xff; 423 pmap_rididx = 0; 424 pmap_ridcount = 8; 425 mutex_init(&pmap_ridmutex, MUTEX_DEFAULT, IPL_VM); 426 427 /* 428 * Compute the number of pages kmem_map will have. 429 */ 430 kmeminit_nkmempages(); 431 432 /* 433 * Figure out how many initial PTE's are necessary to map the 434 * kernel. We also reserve space for kmem_alloc_pageable() 435 * for vm_fork(). 436 */ 437 438 /* Get size of buffer cache and set an upper limit */ 439 bufsz = buf_memcalc(); 440 buf_setvalimit(bufsz); 441 442 nkpt = (((ubc_nwins << ubc_winshift) + 443 bufsz + 16 * NCARGS + pager_map_size) / PAGE_SIZE + 444 USRIOSIZE + (maxproc * UPAGES) + nkmempages) / NKPTEPG; 445 446 /* 447 * Allocate some memory for initial kernel 'page tables'. 448 */ 449 ia64_kptdir = (void *)uvm_pageboot_alloc(PAGE_SIZE); 450 memset((void *)ia64_kptdir, 0, PAGE_SIZE); 451 nkpt = 0; 452 kernel_vm_end = VM_INIT_KERNEL_ADDRESS; 453 454 /* 455 * Determine a valid (mappable) VHPT size. 456 */ 457 if (pmap_vhpt_log2size == 0) 458 pmap_vhpt_log2size = 20; 459 else if (pmap_vhpt_log2size < 16) 460 pmap_vhpt_log2size = 16; 461 else if (pmap_vhpt_log2size > 28) 462 pmap_vhpt_log2size = 28; 463 if (pmap_vhpt_log2size & 1) 464 pmap_vhpt_log2size--; 465 466 size = 1UL << pmap_vhpt_log2size; 467 /* XXX add some retries here */ 468 base = pmap_steal_vhpt_memory(size); 469 470 curcpu()->ci_vhpt = base; 471 472 if (base == 0) 473 panic("Unable to allocate VHPT"); 474 475 pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte); 476 pmap_vhpt_bucket = (void *)uvm_pageboot_alloc(pmap_vhpt_nbuckets * 477 sizeof(struct ia64_bucket)); 478 if (bootverbose) 479 printf("VHPT: address=%#lx, size=%#lx, buckets=%ld, address=%lx\n", 480 base, size, pmap_vhpt_nbuckets, (long unsigned int)&pmap_vhpt_bucket[0]); 481 482 for (i = 0; i < pmap_vhpt_nbuckets; i++) { 483 /* Stolen memory is zeroed. */ 484 mutex_init(&pmap_vhpt_bucket[i].mutex, MUTEX_DEFAULT, IPL_VM); 485 } 486 487 pmap_initialize_vhpt(base); 488 map_vhpt(base); 489 ia64_set_pta(base + (1 << 8) + (pmap_vhpt_log2size << 2) + 1); 490 ia64_srlz_i(); 491 492 /* XXX 493 virtual_avail = VM_INIT_KERNEL_ADDRESS; 494 virtual_end = VM_MAX_KERNEL_ADDRESS; 495 */ 496 497 /* 498 * Initialize the kernel pmap (which is statically allocated). 499 */ 500 PMAP_LOCK_INIT(kernel_pmap); 501 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) 502 kernel_pmap->pm_rid[i] = 0; 503 TAILQ_INIT(&kernel_pmap->pm_pvchunk); 504 505 bzero(&kernel_pmap->pm_stats, sizeof kernel_pmap->pm_stats); 506 kernel_pmap->pm_refcount = 1; 507 508 curcpu()->ci_pmap = kernel_pmap; 509 510 /* 511 * Initialize the global pv list lock. 512 */ 513 rw_init(&pvh_global_lock); 514 515 /* Region 5 is mapped via the VHPT. */ 516 ia64_set_rr(IA64_RR_BASE(5), (5 << 8) | (PAGE_SHIFT << 2) | 1); 517 518 /* 519 * Clear out any random TLB entries left over from booting. 520 */ 521 pmap_invalidate_all(); 522 523 map_gateway_page(); 524 } 525 526 vaddr_t 527 pmap_page_to_va(struct vm_page *m) 528 { 529 paddr_t pa; 530 vaddr_t va; 531 532 pa = VM_PAGE_TO_PHYS(m); 533 va = (m->mdpage.memattr == VM_MEMATTR_UNCACHEABLE) ? IA64_PHYS_TO_RR6(pa) : 534 IA64_PHYS_TO_RR7(pa); 535 return (va); 536 } 537 538 /*************************************************** 539 * Manipulate TLBs for a pmap 540 ***************************************************/ 541 542 static void 543 pmap_invalidate_page(vaddr_t va) 544 { 545 struct ia64_lpte *pte; 546 //struct pcpu *pc; 547 uint64_t tag; 548 u_int vhpt_ofs; 549 struct cpu_info *ci; 550 CPU_INFO_ITERATOR cii; 551 552 critical_enter(); 553 554 vhpt_ofs = ia64_thash(va) - curcpu()->ci_vhpt; 555 556 tag = ia64_ttag(va); 557 558 for (CPU_INFO_FOREACH(cii,ci)) { 559 pte = (struct ia64_lpte *)(ci->ci_vhpt + vhpt_ofs); 560 atomic_cmpset_64(&pte->tag, tag, 1UL << 63); 561 } 562 563 mutex_spin_enter(&pmap_ptc_mutex); 564 565 ia64_ptc_ga(va, PAGE_SHIFT << 2); 566 ia64_mf(); 567 ia64_srlz_i(); 568 569 mutex_spin_exit(&pmap_ptc_mutex); 570 571 ia64_invala(); 572 573 critical_exit(); 574 } 575 576 void 577 pmap_invalidate_all(void) 578 { 579 uint64_t addr; 580 int i, j; 581 582 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 583 584 addr = pmap_ptc_e_base; 585 for (i = 0; i < pmap_ptc_e_count1; i++) { 586 for (j = 0; j < pmap_ptc_e_count2; j++) { 587 ia64_ptc_e(addr); 588 addr += pmap_ptc_e_stride2; 589 } 590 addr += pmap_ptc_e_stride1; 591 } 592 ia64_srlz_i(); 593 } 594 595 static uint32_t 596 pmap_allocate_rid(void) 597 { 598 uint64_t bit, bits; 599 int rid; 600 601 mutex_enter(&pmap_ridmutex); 602 603 if (pmap_ridcount == pmap_ridmax) 604 panic("pmap_allocate_rid: All Region IDs used"); 605 606 /* Find an index with a free bit. */ 607 while ((bits = pmap_ridmap[pmap_rididx]) == ~0UL) { 608 pmap_rididx++; 609 if (pmap_rididx == pmap_ridmapsz) 610 pmap_rididx = 0; 611 } 612 rid = pmap_rididx * 64; 613 614 /* Find a free bit. */ 615 bit = 1UL; 616 while (bits & bit) { 617 rid++; 618 bit <<= 1; 619 } 620 621 pmap_ridmap[pmap_rididx] |= bit; 622 pmap_ridcount++; 623 624 mutex_exit(&pmap_ridmutex); 625 626 return rid; 627 } 628 629 static void 630 pmap_free_rid(uint32_t rid) 631 { 632 uint64_t bit; 633 int idx; 634 635 idx = rid / 64; 636 bit = ~(1UL << (rid & 63)); 637 638 mutex_enter(&pmap_ridmutex); 639 pmap_ridmap[idx] &= bit; 640 pmap_ridcount--; 641 642 mutex_exit(&pmap_ridmutex); 643 } 644 645 /*************************************************** 646 * Page table page management routines..... 647 ***************************************************/ 648 CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); 649 650 static __inline struct pv_chunk * 651 pv_to_chunk(pv_entry_t pv) 652 { 653 return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); 654 } 655 656 #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) 657 658 #define PC_FREE_FULL 0xfffffffffffffffful 659 #define PC_FREE_PARTIAL \ 660 ((1UL << (_NPCPV - sizeof(u_long) * 8 * (_NPCM - 1))) - 1) 661 662 #if PAGE_SIZE == 8192 663 static const u_long pc_freemask[_NPCM] = { 664 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL, 665 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_PARTIAL 666 }; 667 #elif PAGE_SIZE == 16384 668 static const u_long pc_freemask[_NPCM] = { 669 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL, 670 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL, 671 PC_FREE_FULL, PC_FREE_FULL, PC_FREE_FULL, 672 PC_FREE_FULL, PC_FREE_PARTIAL 673 }; 674 #endif 675 676 #ifdef PV_STATS 677 static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; 678 static long pv_entry_frees, pv_entry_allocs; 679 static int pv_entry_spare; 680 #endif 681 682 /* 683 * We are in a serious low memory condition. Resort to 684 * drastic measures to free some pages so we can allocate 685 * another pv entry chunk. 686 */ 687 static struct vm_page 688 *pmap_pv_reclaim(pmap_t locked_pmap) 689 { 690 struct pch newtail; 691 struct pv_chunk *pc; 692 struct ia64_lpte *pte; 693 pmap_t pmap; 694 pv_entry_t pv; 695 vaddr_t va; 696 struct vm_page *m; 697 struct vm_page *m_pc; 698 u_long inuse; 699 int bit, field, freed, idx; 700 701 PMAP_LOCK_ASSERT(locked_pmap); 702 pmap = NULL; 703 m_pc = NULL; 704 TAILQ_INIT(&newtail); 705 while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL) { 706 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 707 if (pmap != pc->pc_pmap) { 708 if (pmap != NULL) { 709 if (pmap != locked_pmap) { 710 pmap_switch(locked_pmap); 711 PMAP_UNLOCK(pmap); 712 } 713 } 714 pmap = pc->pc_pmap; 715 /* Avoid deadlock and lock recursion. */ 716 if (pmap > locked_pmap) 717 PMAP_LOCK(pmap); 718 else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) { 719 pmap = NULL; 720 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); 721 continue; 722 } 723 pmap_switch(pmap); 724 } 725 726 /* 727 * Destroy every non-wired, 8 KB page mapping in the chunk. 728 */ 729 freed = 0; 730 for (field = 0; field < _NPCM; field++) { 731 for (inuse = ~pc->pc_map[field] & pc_freemask[field]; 732 inuse != 0; inuse &= ~(1UL << bit)) { 733 bit = ffs64(inuse) - 1; 734 idx = field * sizeof(inuse) * NBBY + bit; 735 pv = &pc->pc_pventry[idx]; 736 va = pv->pv_va; 737 pte = pmap_find_vhpt(va); 738 KASSERTMSG(pte != NULL, "pte"); 739 if (pmap_wired(pte)) 740 continue; 741 pmap_remove_vhpt(va); 742 pmap_invalidate_page(va); 743 m = PHYS_TO_VM_PAGE(pmap_ppn(pte)); 744 if (pmap_dirty(pte)) 745 vm_page_dirty(m); 746 pmap_free_pte(pte, va); 747 TAILQ_REMOVE(&m->mdpage.pv_list, pv, pv_list); 748 /* XXX 749 if (TAILQ_EMPTY(&m->mdpage.pv_list)) 750 //vm_page_aflag_clear(m, PGA_WRITEABLE); 751 m->flags |= PG_RDONLY; 752 */ 753 pc->pc_map[field] |= 1UL << bit; 754 freed++; 755 } 756 } 757 if (freed == 0) { 758 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); 759 continue; 760 } 761 /* Every freed mapping is for a 8 KB page. */ 762 pmap->pm_stats.resident_count -= freed; 763 PV_STAT(pv_entry_frees += freed); 764 PV_STAT(pv_entry_spare += freed); 765 pv_entry_count -= freed; 766 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 767 for (field = 0; field < _NPCM; field++) 768 if (pc->pc_map[field] != pc_freemask[field]) { 769 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, 770 pc_list); 771 TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); 772 773 /* 774 * One freed pv entry in locked_pmap is 775 * sufficient. 776 */ 777 if (pmap == locked_pmap) 778 goto out; 779 break; 780 } 781 if (field == _NPCM) { 782 PV_STAT(pv_entry_spare -= _NPCPV); 783 PV_STAT(pc_chunk_count--); 784 PV_STAT(pc_chunk_frees++); 785 /* Entire chunk is free; return it. */ 786 m_pc = PHYS_TO_VM_PAGE(IA64_RR_MASK((vaddr_t)pc)); 787 break; 788 } 789 } 790 out: 791 TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); 792 if (pmap != NULL) { 793 if (pmap != locked_pmap) { 794 pmap_switch(locked_pmap); 795 PMAP_UNLOCK(pmap); 796 } 797 } 798 return (m_pc); 799 } 800 801 /* 802 * free the pv_entry back to the free list 803 */ 804 static void 805 free_pv_entry(pmap_t pmap, pv_entry_t pv) 806 { 807 struct pv_chunk *pc; 808 int bit, field, idx; 809 810 KASSERT(rw_write_held(&pvh_global_lock)); 811 PMAP_LOCK_ASSERT(pmap); 812 PV_STAT(pv_entry_frees++); 813 PV_STAT(pv_entry_spare++); 814 pv_entry_count--; 815 pc = pv_to_chunk(pv); 816 idx = pv - &pc->pc_pventry[0]; 817 field = idx / (sizeof(u_long) * NBBY); 818 bit = idx % (sizeof(u_long) * NBBY); 819 pc->pc_map[field] |= 1ul << bit; 820 for (idx = 0; idx < _NPCM; idx++) 821 if (pc->pc_map[idx] != pc_freemask[idx]) { 822 /* 823 * 98% of the time, pc is already at the head of the 824 * list. If it isn't already, move it to the head. 825 */ 826 if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != 827 pc)) { 828 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 829 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, 830 pc_list); 831 } 832 return; 833 } 834 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 835 free_pv_chunk(pc); 836 } 837 838 static void 839 free_pv_chunk(struct pv_chunk *pc) 840 { 841 struct vm_page *m; 842 843 TAILQ_REMOVE(&pv_chunks, pc, pc_lru); 844 PV_STAT(pv_entry_spare -= _NPCPV); 845 PV_STAT(pc_chunk_count--); 846 PV_STAT(pc_chunk_frees++); 847 /* entire chunk is free, return it */ 848 m = PHYS_TO_VM_PAGE(IA64_RR_MASK((vaddr_t)pc)); 849 #if 0 850 /* XXX freebsd these move pages around in queue */ 851 vm_page_unwire(m, 0); // releases one wiring and moves page back active/inactive queue 852 vm_page_free(m); // moves page to "free" queue & disassociate with object 853 854 /* XXX might to need locks/other checks here, uvm_unwire, pmap_kremove... */ 855 uvm_pagefree(m); 856 #endif 857 vm_page_free1(m); 858 } 859 860 /* 861 * get a new pv_entry, allocating a block from the system 862 * when needed. 863 */ 864 static pv_entry_t 865 get_pv_entry(pmap_t pmap, bool try) 866 { 867 struct pv_chunk *pc; 868 pv_entry_t pv; 869 struct vm_page *m; 870 int bit, field, idx; 871 872 KASSERT(rw_write_held(&pvh_global_lock)); 873 PMAP_LOCK_ASSERT(pmap); 874 PV_STAT(pv_entry_allocs++); 875 pv_entry_count++; 876 retry: 877 pc = TAILQ_FIRST(&pmap->pm_pvchunk); 878 if (pc != NULL) { 879 for (field = 0; field < _NPCM; field++) { 880 if (pc->pc_map[field]) { 881 bit = ffs64(pc->pc_map[field]) - 1; 882 break; 883 } 884 } 885 if (field < _NPCM) { 886 idx = field * sizeof(pc->pc_map[field]) * NBBY + bit; 887 pv = &pc->pc_pventry[idx]; 888 pc->pc_map[field] &= ~(1ul << bit); 889 /* If this was the last item, move it to tail */ 890 for (field = 0; field < _NPCM; field++) 891 if (pc->pc_map[field] != 0) { 892 PV_STAT(pv_entry_spare--); 893 return (pv); /* not full, return */ 894 } 895 TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); 896 TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); 897 PV_STAT(pv_entry_spare--); 898 return (pv); 899 } 900 } 901 902 /* No free items, allocate another chunk */ 903 m = vm_page_alloc1(); 904 905 if (m == NULL) { 906 if (try) { 907 pv_entry_count--; 908 PV_STAT(pc_chunk_tryfail++); 909 return (NULL); 910 } 911 m = pmap_pv_reclaim(pmap); 912 if (m == NULL) 913 goto retry; 914 } 915 916 PV_STAT(pc_chunk_count++); 917 PV_STAT(pc_chunk_allocs++); 918 pc = (struct pv_chunk *)IA64_PHYS_TO_RR7(VM_PAGE_TO_PHYS(m)); 919 pc->pc_pmap = pmap; 920 pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */ 921 for (field = 1; field < _NPCM; field++) 922 pc->pc_map[field] = pc_freemask[field]; 923 TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); 924 pv = &pc->pc_pventry[0]; 925 TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); 926 PV_STAT(pv_entry_spare += _NPCPV - 1); 927 return (pv); 928 } 929 930 /* 931 * Conditionally create a pv entry. 932 */ 933 #if 0 934 static bool 935 pmap_try_insert_pv_entry(pmap_t pmap, vaddr_t va, struct vm_page *m) 936 { 937 pv_entry_t pv; 938 939 PMAP_LOCK_ASSERT(pmap); 940 KASSERT(rw_write_held(&pvh_global_lock)); 941 if ((pv = get_pv_entry(pmap, true)) != NULL) { 942 pv->pv_va = va; 943 TAILQ_INSERT_TAIL(&m->mdpage.pv_list, pv, pv_list); 944 return (true); 945 } else 946 return (false); 947 } 948 #endif 949 950 /* 951 * Add an ia64_lpte to the VHPT. 952 */ 953 static void 954 pmap_enter_vhpt(struct ia64_lpte *pte, vaddr_t va) 955 { 956 struct ia64_bucket *bckt; 957 struct ia64_lpte *vhpte; 958 uint64_t pte_pa; 959 960 /* Can fault, so get it out of the way. */ 961 pte_pa = ia64_tpa((vaddr_t)pte); 962 963 vhpte = (struct ia64_lpte *)ia64_thash(va); 964 bckt = (struct ia64_bucket *)vhpte->chain; 965 966 mutex_spin_enter(&bckt->mutex); 967 pte->chain = bckt->chain; 968 ia64_mf(); 969 bckt->chain = pte_pa; 970 971 pmap_vhpt_inserts++; 972 bckt->length++; 973 mutex_spin_exit(&bckt->mutex); 974 } 975 976 /* 977 * Remove the ia64_lpte matching va from the VHPT. Return zero if it 978 * worked or an appropriate error code otherwise. 979 */ 980 static int 981 pmap_remove_vhpt(vaddr_t va) 982 { 983 struct ia64_bucket *bckt; 984 struct ia64_lpte *pte; 985 struct ia64_lpte *lpte; 986 struct ia64_lpte *vhpte; 987 uint64_t chain, tag; 988 989 tag = ia64_ttag(va); 990 vhpte = (struct ia64_lpte *)ia64_thash(va); 991 bckt = (struct ia64_bucket *)vhpte->chain; 992 993 lpte = NULL; 994 mutex_spin_enter(&bckt->mutex); 995 chain = bckt->chain; 996 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 997 while (chain != 0 && pte->tag != tag) { 998 lpte = pte; 999 chain = pte->chain; 1000 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1001 } 1002 if (chain == 0) { 1003 mutex_spin_exit(&bckt->mutex); 1004 return (ENOENT); 1005 } 1006 1007 /* Snip this pv_entry out of the collision chain. */ 1008 if (lpte == NULL) 1009 bckt->chain = pte->chain; 1010 else 1011 lpte->chain = pte->chain; 1012 ia64_mf(); 1013 1014 bckt->length--; 1015 1016 mutex_spin_exit(&bckt->mutex); 1017 return (0); 1018 } 1019 1020 /* 1021 * Find the ia64_lpte for the given va, if any. 1022 */ 1023 static struct ia64_lpte * 1024 pmap_find_vhpt(vaddr_t va) 1025 { 1026 struct ia64_bucket *bckt; 1027 struct ia64_lpte *pte; 1028 uint64_t chain, tag; 1029 1030 tag = ia64_ttag(va); 1031 pte = (struct ia64_lpte *)ia64_thash(va); 1032 bckt = (struct ia64_bucket *)pte->chain; 1033 1034 mutex_spin_enter(&bckt->mutex); 1035 chain = bckt->chain; 1036 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1037 while (chain != 0 && pte->tag != tag) { 1038 chain = pte->chain; 1039 pte = (struct ia64_lpte *)IA64_PHYS_TO_RR7(chain); 1040 } 1041 1042 mutex_spin_exit(&bckt->mutex); 1043 return ((chain != 0) ? pte : NULL); 1044 } 1045 1046 /* 1047 * Remove an entry from the list of managed mappings. 1048 */ 1049 static int 1050 pmap_remove_entry(pmap_t pmap, struct vm_page *m, vaddr_t va, pv_entry_t pv) 1051 { 1052 1053 KASSERT(rw_write_held(&pvh_global_lock)); 1054 if (!pv) { 1055 TAILQ_FOREACH(pv, &m->mdpage.pv_list, pv_list) { 1056 if (pmap == PV_PMAP(pv) && va == pv->pv_va) 1057 break; 1058 } 1059 } 1060 1061 if (pv) { 1062 TAILQ_REMOVE(&m->mdpage.pv_list, pv, pv_list); 1063 /* XXX 1064 if (TAILQ_FIRST(&m->mdpage.pv_list) == NULL) 1065 //vm_page_aflag_clear(m, PGA_WRITEABLE); 1066 m->flags |= PG_RDONLY; 1067 */ 1068 free_pv_entry(pmap, pv); 1069 return 0; 1070 } else { 1071 return ENOENT; 1072 } 1073 } 1074 1075 /* 1076 * Create a pv entry for page at pa for 1077 * (pmap, va). 1078 */ 1079 static void 1080 pmap_insert_entry(pmap_t pmap, vaddr_t va, struct vm_page *m) 1081 { 1082 pv_entry_t pv; 1083 1084 KASSERT(rw_write_held(&pvh_global_lock)); 1085 pv = get_pv_entry(pmap, false); 1086 pv->pv_va = va; 1087 TAILQ_INSERT_TAIL(&m->mdpage.pv_list, pv, pv_list); 1088 } 1089 1090 /*************************************************** 1091 * Low level mapping routines..... 1092 ***************************************************/ 1093 1094 /* 1095 * Find the kernel lpte for mapping the given virtual address, which 1096 * must be in the part of region 5 which we can cover with our kernel 1097 * 'page tables'. 1098 */ 1099 static struct ia64_lpte * 1100 pmap_find_kpte(vaddr_t va) 1101 { 1102 struct ia64_lpte **dir1; 1103 struct ia64_lpte *leaf; 1104 1105 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1106 UVMHIST_LOG(maphist, "(va=%p)", va, 0, 0, 0); 1107 1108 KASSERTMSG((va >> 61) == 5, "kernel mapping 0x%lx not in region 5", va); 1109 1110 KASSERTMSG(va < kernel_vm_end, "kernel mapping 0x%lx out of range", va); 1111 1112 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(va)]; 1113 leaf = dir1[KPTE_DIR1_INDEX(va)]; 1114 1115 UVMHIST_LOG(maphist, "(kpte_dir0=%#lx, kpte_dir1=%#lx, kpte_pte=%#lx)", 1116 KPTE_DIR0_INDEX(va), KPTE_DIR1_INDEX(va), KPTE_PTE_INDEX(va), 0); 1117 UVMHIST_LOG(maphist, "(dir1=%p, leaf=%p ret=%p)", 1118 dir1, leaf, &leaf[KPTE_PTE_INDEX(va)], 0); 1119 1120 return (&leaf[KPTE_PTE_INDEX(va)]); 1121 } 1122 1123 /* 1124 * Find a pte suitable for mapping a user-space address. If one exists 1125 * in the VHPT, that one will be returned, otherwise a new pte is 1126 * allocated. 1127 */ 1128 static struct ia64_lpte * 1129 pmap_find_pte(vaddr_t va) 1130 { 1131 struct ia64_lpte *pte; 1132 1133 if (va >= VM_MAXUSER_ADDRESS) 1134 return pmap_find_kpte(va); 1135 1136 pte = pmap_find_vhpt(va); 1137 if (pte == NULL) { 1138 pte = pool_cache_get(pte_pool_cache, PR_NOWAIT); 1139 if (pte != NULL) { 1140 memset((void *)pte, 0, sizeof(struct ia64_lpte)); 1141 pte->tag = 1UL << 63; 1142 } 1143 } 1144 1145 return (pte); 1146 } 1147 1148 /* 1149 * Free a pte which is now unused. This simply returns it to the zone 1150 * allocator if it is a user mapping. For kernel mappings, clear the 1151 * valid bit to make it clear that the mapping is not currently used. 1152 */ 1153 static void 1154 pmap_free_pte(struct ia64_lpte *pte, vaddr_t va) 1155 { 1156 if (va < VM_MAXUSER_ADDRESS) 1157 pool_cache_put(pte_pool_cache, pte); 1158 else 1159 pmap_clear_present(pte); 1160 } 1161 1162 static PMAP_INLINE void 1163 pmap_pte_prot(pmap_t pm, struct ia64_lpte *pte, vm_prot_t prot) 1164 { 1165 static long prot2ar[4] = { 1166 PTE_AR_R, /* VM_PROT_NONE */ 1167 PTE_AR_RW, /* VM_PROT_WRITE */ 1168 PTE_AR_RX|PTE_ED, /* VM_PROT_EXECUTE */ 1169 PTE_AR_RWX|PTE_ED /* VM_PROT_WRITE|VM_PROT_EXECUTE */ 1170 }; 1171 1172 pte->pte &= ~(PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED); 1173 pte->pte |= (uint64_t)(prot & VM_PROT_ALL) << 56; 1174 pte->pte |= (prot == VM_PROT_NONE || pm == kernel_pmap) 1175 ? PTE_PL_KERN : PTE_PL_USER; 1176 pte->pte |= prot2ar[(prot & VM_PROT_ALL) >> 1]; 1177 } 1178 1179 static PMAP_INLINE void 1180 pmap_pte_attr(struct ia64_lpte *pte, vm_memattr_t ma) 1181 { 1182 pte->pte &= ~PTE_MA_MASK; 1183 pte->pte |= (ma & PTE_MA_MASK); 1184 } 1185 1186 /* 1187 * Set a pte to contain a valid mapping and enter it in the VHPT. If 1188 * the pte was originally valid, then its assumed to already be in the 1189 * VHPT. 1190 * This functions does not set the protection bits. It's expected 1191 * that those have been set correctly prior to calling this function. 1192 */ 1193 static void 1194 pmap_set_pte(struct ia64_lpte *pte, vaddr_t va, vaddr_t pa, 1195 bool wired, bool managed) 1196 { 1197 pte->pte &= PTE_PROT_MASK | PTE_MA_MASK | PTE_PL_MASK | 1198 PTE_AR_MASK | PTE_ED; 1199 pte->pte |= PTE_PRESENT; 1200 pte->pte |= (managed) ? PTE_MANAGED : (PTE_DIRTY | PTE_ACCESSED); 1201 pte->pte |= (wired) ? PTE_WIRED : 0; 1202 pte->pte |= pa & PTE_PPN_MASK; 1203 1204 pte->itir = PAGE_SHIFT << 2; 1205 1206 ia64_mf(); 1207 1208 pte->tag = ia64_ttag(va); 1209 } 1210 1211 /* 1212 * Remove the (possibly managed) mapping represented by pte from the 1213 * given pmap. 1214 */ 1215 static int 1216 pmap_remove_pte(pmap_t pmap, struct ia64_lpte *pte, vaddr_t va, 1217 pv_entry_t pv, int freepte) 1218 { 1219 int error; 1220 struct vm_page *m; 1221 1222 /* 1223 * First remove from the VHPT. 1224 */ 1225 error = pmap_remove_vhpt(va); 1226 KASSERTMSG(error == 0, "%s: pmap_remove_vhpt returned %d",__func__, error); 1227 1228 pmap_invalidate_page(va); 1229 1230 if (pmap_wired(pte)) 1231 pmap->pm_stats.wired_count -= 1; 1232 1233 pmap->pm_stats.resident_count -= 1; 1234 if (pmap_managed(pte)) { 1235 m = PHYS_TO_VM_PAGE(pmap_ppn(pte)); 1236 if (pmap_dirty(pte)) 1237 vm_page_dirty(m); 1238 1239 error = pmap_remove_entry(pmap, m, va, pv); 1240 } 1241 if (freepte) 1242 pmap_free_pte(pte, va); 1243 1244 return (error); 1245 } 1246 1247 /* 1248 * pmap_init: [ INTERFACE ] 1249 * 1250 * Initialize the pmap module. Called by vm_init(), to initialize any 1251 * structures that the pmap system needs to map virtual memory. 1252 * 1253 * Note: no locking is necessary in this function. 1254 */ 1255 void 1256 pmap_init(void) 1257 { 1258 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1259 1260 pmap_pool_cache = pool_cache_init(sizeof(struct pmap), 0, 0, 0, 1261 "pmap_pool_cache", NULL, IPL_VM, 1262 NULL, NULL, NULL); 1263 if (pmap_pool_cache == NULL) 1264 panic("%s cannot allocate pmap pool", __func__); 1265 1266 pte_pool_cache = pool_cache_init(sizeof(struct ia64_lpte), 0, 0, 0, 1267 "pte_pool_cache", NULL, IPL_VM, 1268 NULL, NULL, NULL); 1269 if (pte_pool_cache == NULL) 1270 panic("%s cannot allocate pte pool", __func__); 1271 1272 1273 pmap_initialized = true; 1274 1275 #if DEBUG 1276 if (0) pmap_testout(); 1277 #endif 1278 } 1279 1280 /* 1281 * pmap_virtual_space: [ INTERFACE ] 1282 * 1283 * Define the initial bounds of the kernel virtual address space. 1284 */ 1285 void 1286 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 1287 { 1288 *vstartp = VM_MIN_KERNEL_ADDRESS; 1289 *vendp = VM_MAX_KERNEL_ADDRESS; 1290 } 1291 1292 /* 1293 * pmap_steal_memory: [ INTERFACE ] 1294 * 1295 * Bootstrap memory allocator (alternative to uvm_pageboot_alloc()). 1296 * This function allows for early dynamic memory allocation until the 1297 * virtual memory system has been bootstrapped. After that point, either 1298 * kmem_alloc or malloc should be used. This function works by stealing 1299 * pages from the (to be) managed page pool, then implicitly mapping the 1300 * pages (by using their RR7 addresses) and zeroing them. 1301 * 1302 * It may be used once the physical memory segments have been pre-loaded 1303 * into the vm_physmem[] array. Early memory allocation MUST use this 1304 * interface! This cannot be used after uvm_page_init(), and will 1305 * generate a panic if tried. 1306 * 1307 * Note that this memory will never be freed, and in essence it is wired 1308 * down. 1309 * 1310 * We must adjust *vstartp and/or *vendp iff we use address space 1311 * from the kernel virtual address range defined by pmap_virtual_space(). 1312 * 1313 * Note: no locking is necessary in this function. 1314 */ 1315 vaddr_t 1316 pmap_steal_memory(vsize_t size, vaddr_t *vstartp, vaddr_t *vendp) 1317 { 1318 int npgs; 1319 uvm_physseg_t upm; 1320 vaddr_t va; 1321 paddr_t pa; 1322 1323 size = round_page(size); 1324 npgs = atop(size); 1325 1326 for (upm = uvm_physseg_get_first(); 1327 uvm_physseg_valid_p(upm); 1328 upm = uvm_physseg_get_next(upm)) { 1329 if (uvm.page_init_done == true) 1330 panic("pmap_steal_memory: called _after_ bootstrap"); 1331 1332 if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) || 1333 uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm)) 1334 continue; 1335 1336 if ((uvm_physseg_get_avail_end(upm) - uvm_physseg_get_avail_start(upm)) 1337 < npgs) 1338 continue; 1339 1340 /* 1341 * There are enough pages here; steal them! 1342 */ 1343 pa = ptoa(uvm_physseg_get_start(upm)); 1344 uvm_physseg_unplug(atop(pa), npgs); 1345 1346 va = IA64_PHYS_TO_RR7(pa); 1347 memset((void *)va, 0, size); 1348 pmap_pages_stolen += npgs; 1349 return va; 1350 } 1351 1352 /* 1353 * If we got here, this was no memory left. 1354 */ 1355 panic("pmap_steal_memory: no memory to steal"); 1356 } 1357 1358 /* 1359 * pmap_steal_vhpt_memory: Derived from alpha/pmap.c:pmap_steal_memory() 1360 * Note: This function is not visible outside the pmap module. 1361 * Based on pmap_steal_memory(); 1362 * Assumptions: size is always a power of 2. 1363 * Returns: Allocated memory at a naturally aligned address 1364 */ 1365 static vaddr_t 1366 pmap_steal_vhpt_memory(vsize_t size) 1367 { 1368 int npgs; 1369 uvm_physseg_t upm; 1370 vaddr_t va; 1371 paddr_t pa = 0; 1372 paddr_t vhpt_start = 0, start1, start2, end1, end2; 1373 1374 size = round_page(size); 1375 npgs = atop(size); 1376 1377 for (upm = uvm_physseg_get_first(); 1378 uvm_physseg_valid_p(upm); 1379 upm = uvm_physseg_get_next(upm)) { 1380 if (uvm.page_init_done == true) 1381 panic("pmap_vhpt_steal_memory: called _after_ bootstrap"); 1382 1383 if (uvm_physseg_get_avail_start(upm) != uvm_physseg_get_start(upm) || /* XXX: ??? */ 1384 uvm_physseg_get_avail_start(upm) >= uvm_physseg_get_avail_end(upm)) 1385 continue; 1386 1387 /* Break off a VHPT sized, aligned chunk off this segment. */ 1388 1389 start1 = uvm_physseg_get_avail_start(upm); 1390 1391 /* Align requested start address on requested size boundary */ 1392 end1 = vhpt_start = roundup(start1, npgs); 1393 1394 start2 = vhpt_start + npgs; 1395 end2 = uvm_physseg_get_avail_end(upm); 1396 1397 /* Case 1: Doesn't fit. skip this segment */ 1398 1399 if (start2 > end2) { 1400 vhpt_start = 0; 1401 continue; 1402 } 1403 1404 /* For all cases of fit: 1405 * - Remove segment. 1406 * - Re-insert fragments via uvm_page_physload(); 1407 */ 1408 1409 /* 1410 * We _fail_ on a vhpt request which exhausts memory. 1411 */ 1412 if (start1 == end1 && 1413 start2 == end2 && 1414 uvm_physseg_get_first() == uvm_physseg_get_last() /* single segment */) { 1415 #ifdef DEBUG 1416 printf("pmap_vhpt_steal_memory: out of memory!"); 1417 #endif 1418 return -1; 1419 } 1420 1421 /* Remove this segment from the list. */ 1422 if (uvm_physseg_unplug(uvm_physseg_get_start(upm), 1423 uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm)) == false) { 1424 panic("%s: uvm_physseg_unplug(%"PRIxPADDR", %"PRIxPADDR") failed\n", 1425 __func__, uvm_physseg_get_start(upm), 1426 uvm_physseg_get_end(upm) - uvm_physseg_get_start(upm)); 1427 } 1428 1429 /* Case 2: Perfect fit - skip segment reload. */ 1430 1431 if (start1 == end1 && start2 == end2) break; 1432 1433 /* Case 3: Left unfit - reload it. 1434 */ 1435 1436 if (start1 != end1) 1437 uvm_page_physload(start1, end1, start1, end1, 1438 VM_FREELIST_DEFAULT); 1439 1440 /* Case 4: Right unfit - reload it. */ 1441 1442 if (start2 != end2) 1443 uvm_page_physload(start2, end2, start2, end2, 1444 VM_FREELIST_DEFAULT); 1445 1446 /* Case 5: Both unfit - Redundant, isn't it ? */ 1447 break; 1448 } 1449 1450 /* 1451 * If we got here, we couldn't find a fit. 1452 */ 1453 if (vhpt_start == 0) { 1454 #ifdef DEBUG 1455 printf("pmap_steal_vhpt_memory: no VHPT aligned fit found."); 1456 #endif 1457 return -1; 1458 } 1459 1460 /* 1461 * There are enough pages here; steal them! 1462 */ 1463 pa = ptoa(vhpt_start); 1464 va = IA64_PHYS_TO_RR7(pa); 1465 memset((void *)va, 0, size); 1466 pmap_pages_stolen += npgs; 1467 return va; 1468 } 1469 1470 /* 1471 * pmap_create: [ INTERFACE ] 1472 * 1473 * Create and return a physical map. 1474 * 1475 * Note: no locking is necessary in this function. 1476 */ 1477 pmap_t 1478 pmap_create(void) 1479 { 1480 pmap_t pmap; 1481 int i; 1482 1483 pmap = pool_cache_get(pmap_pool_cache, PR_WAITOK); 1484 1485 if (pmap == NULL) 1486 panic("%s no pool", __func__); 1487 1488 PMAP_LOCK_INIT(pmap); 1489 1490 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) 1491 pmap->pm_rid[i] = pmap_allocate_rid(); 1492 1493 TAILQ_INIT(&pmap->pm_pvchunk); 1494 1495 pmap->pm_stats.resident_count = 0; 1496 pmap->pm_stats.wired_count = 0; 1497 1498 pmap->pm_refcount = 1; 1499 1500 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1501 UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0); 1502 1503 return pmap; 1504 } 1505 1506 /* 1507 * pmap_destroy: [ INTERFACE ] 1508 * 1509 * Drop the reference count on the specified pmap, releasing 1510 * all resources if the reference count drops to zero. 1511 */ 1512 void 1513 pmap_destroy(pmap_t pmap) 1514 { 1515 int i; 1516 1517 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1518 UVMHIST_LOG(maphist, "(pm=%p)", pmap, 0, 0, 0); 1519 1520 membar_release(); 1521 if (atomic_dec_64_nv(&pmap->pm_refcount) > 0) 1522 return; 1523 membar_acquire(); 1524 1525 KASSERT(pmap->pm_stats.resident_count == 0); 1526 KASSERT(pmap->pm_stats.wired_count == 0); 1527 1528 KASSERT(TAILQ_EMPTY(&pmap->pm_pvchunk)); /* XXX hmmm */ 1529 KASSERT(!PMAP_LOCKED(pmap)); /* XXX hmmm */ 1530 /*PMAP_LOCK(pmap); */ /* XXX overkill */ 1531 1532 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) 1533 if (pmap->pm_rid[i]) 1534 pmap_free_rid(pmap->pm_rid[i]); 1535 1536 /*PMAP_UNLOCK(pmap);*/ /* XXX hmm */ 1537 PMAP_LOCK_DESTROY(pmap); 1538 1539 pool_cache_put(pmap_pool_cache, pmap); 1540 } 1541 1542 /* 1543 * pmap_reference: [ INTERFACE ] 1544 * 1545 * Add a reference to the specified pmap. 1546 */ 1547 void 1548 pmap_reference(pmap_t pmap) 1549 { 1550 atomic_inc_64(&pmap->pm_refcount); 1551 } 1552 1553 /* 1554 * pmap_resident_count: [ INTERFACE ] 1555 * 1556 * Query the ``resident pages'' statistic for pmap. 1557 */ 1558 long 1559 pmap_resident_count(pmap_t pmap) 1560 { 1561 return (pmap->pm_stats.resident_count); 1562 } 1563 1564 /* 1565 * pmap_wired_count: [ INTERFACE ] 1566 * 1567 * Query the ``wired pages'' statistic for pmap. 1568 * 1569 */ 1570 long 1571 pmap_wired_count(pmap_t pmap) 1572 { 1573 return (pmap->pm_stats.wired_count); 1574 } 1575 1576 /* 1577 * pmap_growkernel: [ INTERFACE ] 1578 * 1579 */ 1580 vaddr_t 1581 pmap_growkernel(vaddr_t maxkvaddr) 1582 { 1583 struct ia64_lpte **dir1; 1584 struct ia64_lpte *leaf; 1585 struct vm_page *pg; 1586 #if 0 1587 struct vm_page *nkpg; 1588 paddr_t pa; 1589 vaddr_t va; 1590 #endif 1591 1592 /* XXX this function may still need serious fixin' */ 1593 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1594 UVMHIST_LOG(maphist, "(va=%#lx, nkpt=%ld, kvm_end=%#lx before)", maxkvaddr, nkpt, kernel_vm_end, 0); 1595 1596 vaddr_t addr = maxkvaddr; 1597 1598 /* XXX use uvm_pageboot_alloc if not done? */ 1599 if (!uvm.page_init_done) 1600 panic("uvm_page init not done"); 1601 1602 while (kernel_vm_end <= addr) { 1603 if (nkpt == PAGE_SIZE/8 + PAGE_SIZE*PAGE_SIZE/64) 1604 panic("%s: out of kernel address space", __func__); 1605 dir1 = ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)]; 1606 1607 if (dir1 == NULL) { 1608 #if 0 1609 /* FreeBSD does it this way... */ 1610 nkpg = vm_page_alloc(NULL, nkpt++, VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED); 1611 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO); 1612 #endif 1613 pg = vm_page_alloc1(); 1614 if (!pg) 1615 panic("%s: cannot add dir1 page", __func__); 1616 nkpt++; 1617 1618 #if 0 1619 dir1 = (struct ia64_lpte **)pmap_page_to_va(nkpg); 1620 bzero(dir1, PAGE_SIZE); 1621 #endif 1622 dir1 = (struct ia64_lpte **)pmap_page_to_va(pg); 1623 1624 ia64_kptdir[KPTE_DIR0_INDEX(kernel_vm_end)] = dir1; 1625 } 1626 1627 #if 0 1628 nkpg = vm_page_alloc(NULL, nkpt++, VM_ALLOC_NOOBJ|VM_ALLOC_INTERRUPT|VM_ALLOC_WIRED); 1629 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO); 1630 #endif 1631 pg = vm_page_alloc1(); 1632 if (!pg) 1633 panic("%s: cannot add PTE page", __func__); 1634 nkpt++; 1635 #if 0 1636 leaf = (struct ia64_lpte *)pmap_page_to_va(nkpg); 1637 bzero(leaf, PAGE_SIZE); 1638 #endif 1639 leaf = (struct ia64_lpte *)pmap_page_to_va(pg); 1640 1641 dir1[KPTE_DIR1_INDEX(kernel_vm_end)] = leaf; 1642 1643 kernel_vm_end += PAGE_SIZE * NKPTEPG; 1644 } 1645 1646 UVMHIST_LOG(maphist, "(va=%#lx, nkpt=%ld, kvm_end=%#lx after)", maxkvaddr, nkpt, kernel_vm_end, 0); 1647 1648 /* XXX fix */ 1649 return kernel_vm_end; 1650 } 1651 1652 /* 1653 * pmap_enter: [ INTERFACE ] 1654 * 1655 * Create a mapping in physical map pmap for the physical 1656 * address pa at the virtual address va with protection speci- 1657 * fied by bits in prot. 1658 */ 1659 int 1660 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1661 { 1662 pmap_t oldpmap; 1663 struct vm_page *m; 1664 vaddr_t opa; 1665 struct ia64_lpte origpte; 1666 struct ia64_lpte *pte; 1667 bool icache_inval, managed, wired, canfail; 1668 /* vm_memattr_t ma; */ 1669 1670 /* XXX this needs work */ 1671 1672 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1673 UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, pa=%p, prot=%#x)", pmap, va, pa, prot); 1674 1675 /* wired = (flags & PMAP_ENTER_WIRED) != 0; */ 1676 wired = (flags & PMAP_WIRED) != 0; 1677 canfail = (flags & PMAP_CANFAIL) != 0; 1678 1679 /* ma = pmap_flags_to_memattr(flags); */ 1680 1681 rw_enter(&pvh_global_lock, RW_WRITER); 1682 PMAP_LOCK(pmap); 1683 oldpmap = pmap_switch(pmap); 1684 1685 va &= ~PAGE_MASK; 1686 KASSERTMSG(va <= VM_MAX_KERNEL_ADDRESS, "%s: toobig", __func__); 1687 1688 /* XXX 1689 if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) 1690 VM_OBJECT_ASSERT_LOCKED(m->object); 1691 */ 1692 1693 /* 1694 * Find (or create) a pte for the given mapping. 1695 */ 1696 pte = pmap_find_pte(va); 1697 1698 if (pte == NULL) { 1699 pmap_switch(oldpmap); 1700 PMAP_UNLOCK(pmap); 1701 rw_exit(&pvh_global_lock); 1702 1703 if (canfail) 1704 return (ENOMEM); 1705 else 1706 panic("%s: no pte available", __func__); 1707 } 1708 1709 origpte = *pte; 1710 if (!pmap_present(pte)) { 1711 opa = ~0UL; 1712 pmap_enter_vhpt(pte, va); 1713 } else 1714 opa = pmap_ppn(pte); 1715 1716 managed = false; 1717 /* XXX hmm 1718 pa = VM_PAGE_TO_PHYS(m); 1719 */ 1720 1721 m = PHYS_TO_VM_PAGE(pa); 1722 if (m == NULL) { 1723 /* implies page not managed? */ 1724 panic("%s: new page needed", __func__); 1725 } 1726 icache_inval = (prot & VM_PROT_EXECUTE) ? true : false; 1727 1728 /* 1729 * Mapping has not changed, must be protection or wiring change. 1730 */ 1731 if (opa == pa) { 1732 /* 1733 * Wiring change, just update stats. We don't worry about 1734 * wiring PT pages as they remain resident as long as there 1735 * are valid mappings in them. Hence, if a user page is wired, 1736 * the PT page will be also. 1737 */ 1738 if (wired && !pmap_wired(&origpte)) 1739 pmap->pm_stats.wired_count++; 1740 else if (!wired && pmap_wired(&origpte)) 1741 pmap->pm_stats.wired_count--; 1742 1743 managed = (pmap_managed(&origpte)) ? true : false; 1744 1745 /* 1746 * We might be turning off write access to the page, 1747 * so we go ahead and sense modify status. Otherwise, 1748 * we can avoid I-cache invalidation if the page 1749 * already allowed execution. 1750 */ 1751 if (managed && pmap_dirty(&origpte)) 1752 vm_page_dirty(m); 1753 else if (pmap_exec(&origpte)) 1754 icache_inval = false; 1755 1756 pmap_invalidate_page(va); 1757 goto validate; 1758 } 1759 1760 /* 1761 * Mapping has changed, invalidate old range and fall 1762 * through to handle validating new mapping. 1763 */ 1764 if (opa != ~0UL) { 1765 pmap_remove_pte(pmap, pte, va, 0, 0); 1766 pmap_enter_vhpt(pte, va); 1767 } 1768 1769 /* 1770 * Enter on the PV list if part of our managed memory. 1771 */ 1772 if (vm_page_is_managed(m)) { 1773 #if 0 1774 KASSERTMSG(va < kmi.clean_sva || va >= kmi.clean_eva, 1775 ("pmap_enter: managed mapping within the clean submap")); 1776 #endif 1777 pmap_insert_entry(pmap, va, m); 1778 managed = true; 1779 } 1780 1781 /* 1782 * Increment counters 1783 */ 1784 pmap->pm_stats.resident_count++; 1785 if (wired) 1786 pmap->pm_stats.wired_count++; 1787 1788 validate: 1789 1790 /* 1791 * Now validate mapping with desired protection/wiring. This 1792 * adds the pte to the VHPT if necessary. 1793 */ 1794 pmap_pte_prot(pmap, pte, prot); 1795 pmap_pte_attr(pte, m->mdpage.memattr); 1796 pmap_set_pte(pte, va, pa, wired, managed); 1797 1798 /* Invalidate the I-cache when needed. */ 1799 if (icache_inval) 1800 ia64_sync_icache(va, PAGE_SIZE); 1801 1802 /* XXX 1803 if ((prot & VM_PROT_WRITE) != 0 && managed) 1804 //vm_page_aflag_set(m, PGA_WRITEABLE); 1805 m->flags &= ~PG_RDONLY; 1806 */ 1807 rw_exit(&pvh_global_lock); 1808 pmap_switch(oldpmap); 1809 PMAP_UNLOCK(pmap); 1810 return (0); 1811 } 1812 1813 /* 1814 * pmap_remove: [ INTERFACE ] 1815 * 1816 * Remove the given range of addresses from the specified map. 1817 * 1818 * It is assumed that the start and end are properly 1819 * rounded to the page size. 1820 * 1821 * Sparsely used ranges are inefficiently removed. The VHPT is 1822 * probed for every page within the range. XXX 1823 */ 1824 void 1825 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1826 { 1827 pmap_t oldpmap; 1828 vaddr_t va; 1829 struct ia64_lpte *pte; 1830 1831 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1832 UVMHIST_LOG(maphist, "(pm=%p, sva=%#lx, eva=%#lx)", pmap, sva, eva, 0); 1833 1834 /* 1835 * Perform an unsynchronized read. This is, however, safe. 1836 */ 1837 if (pmap->pm_stats.resident_count == 0) 1838 return; 1839 1840 rw_enter(&pvh_global_lock, RW_WRITER); 1841 PMAP_LOCK(pmap); 1842 oldpmap = pmap_switch(pmap); 1843 for (va = sva; va < eva; va += PAGE_SIZE) { 1844 pte = pmap_find_vhpt(va); 1845 if (pte != NULL) 1846 pmap_remove_pte(pmap, pte, va, 0, 1); 1847 } 1848 1849 rw_exit(&pvh_global_lock); 1850 pmap_switch(oldpmap); 1851 PMAP_UNLOCK(pmap); 1852 } 1853 1854 /* 1855 * pmap_remove_all: [ INTERFACE ] 1856 * 1857 * This function is a hint to the pmap implementation that all 1858 * entries in pmap will be removed before any more entries are 1859 * entered. 1860 */ 1861 bool 1862 pmap_remove_all(pmap_t pmap) 1863 { 1864 /* XXX do nothing */ 1865 return false; 1866 } 1867 1868 /* 1869 * pmap_protect: [ INTERFACE ] 1870 * 1871 * Set the physical protection on the specified range of this map 1872 * as requested. 1873 */ 1874 void 1875 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1876 { 1877 pmap_t oldpmap; 1878 struct ia64_lpte *pte; 1879 1880 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1881 UVMHIST_LOG(maphist, "(pm=%p, sva=%#lx, eva=%#lx, prot=%#x)", 1882 pmap, sva, eva, prot); 1883 UVMHIST_LOG(maphist, "(VM_PROT_READ=%u, VM_PROT_WRITE=%u, VM_PROT_EXECUTE=%u)", 1884 (prot & VM_PROT_READ) != 0, (prot & VM_PROT_WRITE) != 0, 1885 (prot & VM_PROT_EXECUTE) != 0, 0); 1886 1887 if ((prot & VM_PROT_READ) == VM_PROT_NONE) { 1888 pmap_remove(pmap, sva, eva); 1889 return; 1890 } 1891 1892 if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == 1893 (VM_PROT_WRITE|VM_PROT_EXECUTE)) 1894 return; 1895 1896 if ((sva & PAGE_MASK) || (eva & PAGE_MASK)) 1897 panic("pmap_protect: unaligned addresses"); 1898 sva = trunc_page(sva); 1899 eva = round_page(eva) - 1; 1900 1901 PMAP_LOCK(pmap); 1902 oldpmap = pmap_switch(pmap); 1903 for ( ; sva < eva; sva += PAGE_SIZE) { 1904 /* If page is invalid, skip this page */ 1905 pte = pmap_find_vhpt(sva); 1906 if (pte == NULL) 1907 continue; 1908 1909 /* If there's no change, skip it too */ 1910 if (pmap_prot(pte) == prot) 1911 continue; 1912 1913 /* wired pages unaffected by prot changes */ 1914 if (pmap_wired(pte)) 1915 continue; 1916 1917 if ((prot & VM_PROT_WRITE) == 0 && 1918 pmap_managed(pte) && pmap_dirty(pte)) { 1919 paddr_t pa = pmap_ppn(pte); 1920 struct vm_page *m = PHYS_TO_VM_PAGE(pa); 1921 1922 vm_page_dirty(m); 1923 pmap_clear_dirty(pte); 1924 } 1925 1926 if (prot & VM_PROT_EXECUTE) 1927 ia64_sync_icache(sva, PAGE_SIZE); 1928 1929 pmap_pte_prot(pmap, pte, prot); 1930 pmap_invalidate_page(sva); 1931 } 1932 pmap_switch(oldpmap); 1933 PMAP_UNLOCK(pmap); 1934 } 1935 1936 /* 1937 * pmap_unwire: [ INTERFACE ] 1938 * 1939 * Clear the wired attribute for a map/virtual-address pair. 1940 * 1941 * The wired attribute of the page table entry is not a hardware feature, 1942 * so there is no need to invalidate any TLB entries. 1943 * 1944 * The mapping must already exist in the pmap. 1945 */ 1946 void 1947 pmap_unwire(pmap_t pmap, vaddr_t va) 1948 { 1949 pmap_t oldpmap; 1950 struct ia64_lpte *pte; 1951 1952 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1953 UVMHIST_LOG(maphist, "(pm=%p, va=%#x)", pmap, va, 0, 0); 1954 1955 PMAP_LOCK(pmap); 1956 oldpmap = pmap_switch(pmap); 1957 1958 pte = pmap_find_vhpt(va); 1959 1960 /* XXX panic if no pte or not wired? */ 1961 if (pte == NULL) 1962 panic("pmap_unwire: %lx not found in vhpt", va); 1963 1964 if (!pmap_wired(pte)) 1965 panic("pmap_unwire: pte %p isn't wired", pte); 1966 1967 pmap->pm_stats.wired_count--; 1968 pmap_clear_wired(pte); 1969 1970 pmap_switch(oldpmap); 1971 PMAP_UNLOCK(pmap); 1972 } 1973 1974 /* 1975 * pmap_extract: [ INTERFACE ] 1976 * 1977 * Extract the physical address associated with the given 1978 * pmap/virtual address pair. 1979 */ 1980 bool 1981 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1982 { 1983 struct ia64_lpte *pte; 1984 pmap_t oldpmap; 1985 paddr_t pa; 1986 1987 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1988 UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, pap=%#lx)", pmap, va, pap, 0); 1989 1990 pa = 0; 1991 1992 PMAP_LOCK(pmap); 1993 oldpmap = pmap_switch(pmap); 1994 pte = pmap_find_vhpt(va); 1995 if (pte != NULL && pmap_present(pte)) 1996 pa = pmap_ppn(pte); 1997 pmap_switch(oldpmap); 1998 PMAP_UNLOCK(pmap); 1999 2000 if (pa && (pap != NULL)) 2001 *pap = pa; 2002 2003 UVMHIST_LOG(maphist, "(pa=%#lx)", pa, 0, 0, 0); 2004 2005 return (pa != 0); 2006 } 2007 /* 2008 * Extract the physical page address associated with a kernel 2009 * virtual address. 2010 */ 2011 paddr_t 2012 pmap_kextract(vaddr_t va) 2013 { 2014 struct ia64_lpte *pte; 2015 /*uint64_t *pbvm_pgtbl;*/ 2016 paddr_t pa; 2017 /*u_int idx;*/ 2018 2019 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2020 UVMHIST_LOG(maphist, "(va=%#lx)", va, 0, 0, 0); 2021 2022 KASSERTMSG(va >= VM_MAXUSER_ADDRESS, "Must be kernel VA"); 2023 2024 /* Regions 6 and 7 are direct mapped. */ 2025 if (va >= IA64_RR_BASE(6)) { 2026 pa = IA64_RR_MASK(va); 2027 goto out; 2028 } 2029 2030 /* Region 5 is our KVA. Bail out if the VA is beyond our limits. */ 2031 if (va >= kernel_vm_end) 2032 goto err_out; 2033 if (va >= VM_INIT_KERNEL_ADDRESS) { 2034 pte = pmap_find_kpte(va); 2035 pa = pmap_present(pte) ? pmap_ppn(pte) | (va & PAGE_MASK) : 0; 2036 goto out; 2037 } 2038 #if 0 /* XXX fix */ 2039 /* The PBVM page table. */ 2040 if (va >= IA64_PBVM_PGTBL + bootinfo->bi_pbvm_pgtblsz) 2041 goto err_out; 2042 if (va >= IA64_PBVM_PGTBL) { 2043 pa = (va - IA64_PBVM_PGTBL) + bootinfo->bi_pbvm_pgtbl; 2044 goto out; 2045 } 2046 2047 /* The PBVM itself. */ 2048 if (va >= IA64_PBVM_BASE) { 2049 pbvm_pgtbl = (void *)IA64_PBVM_PGTBL; 2050 idx = (va - IA64_PBVM_BASE) >> IA64_PBVM_PAGE_SHIFT; 2051 if (idx >= (bootinfo->bi_pbvm_pgtblsz >> 3)) 2052 goto err_out; 2053 if ((pbvm_pgtbl[idx] & PTE_PRESENT) == 0) 2054 goto err_out; 2055 pa = (pbvm_pgtbl[idx] & PTE_PPN_MASK) + 2056 (va & IA64_PBVM_PAGE_MASK); 2057 goto out; 2058 } 2059 #endif 2060 2061 err_out: 2062 KASSERT(1); 2063 printf("XXX: %s: va=%#lx is invalid\n", __func__, va); 2064 pa = 0; 2065 /* FALLTHROUGH */ 2066 2067 out: 2068 UVMHIST_LOG(maphist, "(pa=%#lx)", pa, 0, 0, 0); 2069 return (pa); 2070 } 2071 2072 /* 2073 * pmap_kenter_pa: [ INTERFACE ] 2074 * 2075 * Enter a va -> pa mapping into the kernel pmap without any 2076 * physical->virtual tracking. 2077 * 2078 * Note: no locking is necessary in this function. 2079 */ 2080 void 2081 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 2082 { 2083 struct ia64_lpte *pte; 2084 vm_memattr_t attr; 2085 const bool managed = false; /* don't gather ref/mod info */ 2086 const bool wired = true; /* pmap_kenter_pa always wired */ 2087 2088 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2089 UVMHIST_LOG(maphist, "(va=%#lx, pa=%#lx, prot=%p, flags=%p)", va, pa, prot, flags); 2090 2091 KASSERT(va >= VM_MIN_KERNEL_ADDRESS && va < VM_MAX_KERNEL_ADDRESS); 2092 2093 attr = pmap_flags_to_memattr(flags); 2094 2095 pte = pmap_find_kpte(va); 2096 if (pmap_present(pte)) 2097 pmap_invalidate_page(va); 2098 else 2099 pmap_enter_vhpt(pte, va); 2100 2101 pmap_pte_prot(kernel_pmap, pte, VM_PROT_ALL); 2102 pmap_pte_attr(pte, attr); 2103 pmap_set_pte(pte, va, pa, wired, managed); 2104 } 2105 2106 /* 2107 * pmap_kremove: [ INTERFACE ] 2108 * 2109 * Remove all mappings starting at virtual address va for size 2110 * bytes from the kernel physical map. All mappings that are 2111 * removed must be the ``unmanaged'' type created with 2112 * pmap_kenter_pa(). The implementation may assert this. 2113 */ 2114 void 2115 pmap_kremove(vaddr_t va, vsize_t size) 2116 { 2117 struct ia64_lpte *pte; 2118 vaddr_t eva = va + size; 2119 2120 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2121 UVMHIST_LOG(maphist, "(va=%#lx)", va, 0, 0, 0); 2122 2123 while (va < eva) { 2124 pte = pmap_find_kpte(va); 2125 if (pmap_present(pte)) { 2126 KASSERT(pmap_managed(pte) != 0); 2127 pmap_remove_vhpt(va); 2128 pmap_invalidate_page(va); 2129 pmap_clear_present(pte); 2130 } 2131 va += PAGE_SIZE; 2132 } 2133 } 2134 2135 /* 2136 * pmap_copy: [ INTERFACE ] 2137 * 2138 * This function copies the mappings starting at src_addr in 2139 * src_map for len bytes into dst_map starting at dst_addr. 2140 * 2141 * Note that while this function is required to be provided by 2142 * a pmap implementation, it is not actually required to do 2143 * anything. pmap_copy() is merely advisory (it is used in 2144 * the fork(2) path to ``pre-fault'' the child's address 2145 * space). 2146 */ 2147 void 2148 pmap_copy(pmap_t dst_map, pmap_t src_map, vaddr_t dst_addr, vsize_t len, 2149 vaddr_t src_addr) 2150 { 2151 /* nothing required */ 2152 } 2153 2154 /* 2155 * pmap_update: [ INTERFACE ] 2156 * 2157 * If a pmap implementation does not delay virtual-to-physical 2158 * mapping updates, pmap_update() has no operation. 2159 */ 2160 void 2161 pmap_update(pmap_t pmap) 2162 { 2163 /* nothing required */ 2164 } 2165 2166 /* 2167 * pmap_activate: [ INTERFACE ] 2168 * 2169 * Activate the pmap used by the specified process. This includes 2170 * reloading the MMU context if the current process, and marking 2171 * the pmap in use by the processor. 2172 * 2173 * Note: We may use only spin locks here, since we are called 2174 * by a critical section in cpu_switch()! 2175 */ 2176 void 2177 pmap_activate(struct lwp *l) 2178 { 2179 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2180 UVMHIST_LOG(maphist, "(lwp=%p)", l, 0, 0, 0); 2181 KASSERT(l == curlwp); 2182 #if 0 2183 /* pmap_switch(vmspace_pmap(td->td_proc->p_vmspace)); */ 2184 pmap_switch(vm_map_pmap(&l->l_proc->p_vmspace->vm_map)); 2185 #else 2186 struct pmap *pmap = l->l_proc->p_vmspace->vm_map.pmap; 2187 2188 if (pmap == pmap_kernel()) { 2189 return; 2190 } 2191 2192 if (l != curlwp) { 2193 return; 2194 } 2195 2196 pmap_switch(pmap); 2197 #endif 2198 } 2199 2200 /* 2201 * pmap_deactivate: [ INTERFACE ] 2202 * 2203 * Deactivate the physical map used by the process behind lwp 2204 * l. It is generally used in conjunction with 2205 * pmap_activate(). Like pmap_activate(), pmap_deactivate() 2206 * may not always be called when l is the current lwp. 2207 * 2208 */ 2209 void 2210 pmap_deactivate(struct lwp *l) 2211 { 2212 /* XXX ? */ 2213 } 2214 2215 /* 2216 * pmap_zero_page: [ INTERFACE ] 2217 * 2218 * Zero the specified (machine independent) page by mapping the page 2219 * into virtual memory and clear its contents, one machine dependent 2220 * page at a time. 2221 * 2222 * Note: no locking is necessary in this function. 2223 */ 2224 void 2225 pmap_zero_page(paddr_t phys) 2226 { 2227 struct vm_page *m; 2228 vaddr_t va; 2229 2230 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2231 UVMHIST_LOG(maphist, "(pa=%p)", phys, 0, 0, 0); 2232 2233 m = PHYS_TO_VM_PAGE(phys); 2234 KASSERT(m != NULL); 2235 2236 va = pmap_page_to_va(m); 2237 KASSERT(trunc_page(va) == va); 2238 2239 UVMHIST_LOG(maphist, "(pa=%p, va=%p)", phys, va, 0, 0); 2240 memset((void *)va, 0, PAGE_SIZE); 2241 } 2242 2243 /* 2244 * pmap_copy_page: [ INTERFACE ] 2245 * 2246 * Copy the specified (machine independent) page by mapping the page 2247 * into virtual memory and using memcpy to copy the page, one machine 2248 * dependent page at a time. 2249 * 2250 * Note: no locking is necessary in this function. 2251 */ 2252 void 2253 pmap_copy_page(paddr_t psrc, paddr_t pdst) 2254 { 2255 struct vm_page *md, *ms; 2256 vaddr_t dst_va, src_va; 2257 2258 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2259 UVMHIST_LOG(maphist, "(sp=%p, dp=%p)", psrc, pdst, 0, 0); 2260 2261 md = PHYS_TO_VM_PAGE(pdst); 2262 ms = PHYS_TO_VM_PAGE(psrc); 2263 KASSERT(md != NULL && ms != NULL); 2264 2265 dst_va = pmap_page_to_va(md); 2266 src_va = pmap_page_to_va(ms); 2267 KASSERT(trunc_page(dst_va) == dst_va && trunc_page(src_va) == src_va); 2268 2269 memcpy((void *)dst_va, (void *)src_va, PAGE_SIZE); 2270 } 2271 2272 /* 2273 * pmap_page_protect: [ INTERFACE ] 2274 * 2275 * Lower the permissions for all mappings of the page pg to 2276 * prot. This function is used by the virtual memory system 2277 * to implement copy-on-write (called with VM_PROT_READ set in 2278 * prot) and to revoke all mappings when cleaning a page 2279 * (called with no bits set in prot). Access permissions must 2280 * never be added to a page as a result of this call. 2281 */ 2282 void 2283 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 2284 { 2285 //struct ia64_lpte *pte; 2286 pmap_t pmap; 2287 pv_entry_t pv; 2288 vaddr_t va; 2289 2290 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2291 UVMHIST_LOG(maphist, "(m=%p, prot=%p)", pg, prot, 0, 0); 2292 2293 if ((prot & VM_PROT_WRITE) != 0) 2294 return; 2295 if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { 2296 /* XXX FreeBSD 2297 if ((pg->mdpage.aflags & PGA_WRITEABLE) == 0) 2298 return; 2299 */ 2300 if (pg->flags & PG_RDONLY) 2301 return; 2302 rw_enter(&pvh_global_lock, RW_WRITER); 2303 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) { 2304 pmap = PV_PMAP(pv); 2305 va = pv->pv_va; 2306 // locking of pmap done in pmap_protect 2307 pmap_protect(pmap, va, va + PAGE_SIZE, prot); 2308 } 2309 /* XXX 2310 vm_page_aflag_clear(pg, PGA_WRITEABLE); 2311 */ 2312 pg->flags |= PG_RDONLY; 2313 2314 rw_exit(&pvh_global_lock); 2315 } else { 2316 pmap_remove_all_phys(pg); 2317 } 2318 } 2319 2320 /* 2321 * pmap_clear_modify: [ INTERFACE ] 2322 * 2323 * Clear the modify bits on the specified physical page. 2324 */ 2325 bool 2326 pmap_clear_modify(struct vm_page *pg) 2327 { 2328 struct ia64_lpte *pte; 2329 pmap_t oldpmap, pmap; 2330 pv_entry_t pv; 2331 bool rv; 2332 2333 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2334 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0); 2335 2336 KASSERTMSG(vm_page_is_managed(pg), "%s : page %p not managed", 2337 __func__, pg); 2338 2339 rv = false; 2340 2341 //VM_OBJECT_ASSERT_WLOCKED(m->object); 2342 //KASSERT(!vm_page_xbusied(m), 2343 // ("pmap_clear_modify: page %p is exclusive busied", m)); 2344 2345 /* 2346 * If the page is not PGA_WRITEABLE, then no PTEs can be modified. 2347 * If the object containing the page is locked and the page is not 2348 * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. 2349 */ 2350 2351 #if 0 /* XXX freebsd does this, looks faster but not required */ 2352 if ((m->aflags & PGA_WRITEABLE) == 0) 2353 return; 2354 if (pg->flags & PG_RDONLY) 2355 return (rv); 2356 #endif 2357 rw_enter(&pvh_global_lock, RW_WRITER); 2358 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) { 2359 pmap = PV_PMAP(pv); 2360 PMAP_LOCK(pmap); 2361 oldpmap = pmap_switch(pmap); 2362 pte = pmap_find_vhpt(pv->pv_va); 2363 KASSERTMSG(pte != NULL, "pte"); 2364 if (pmap_dirty(pte)) { 2365 pmap_clear_dirty(pte); 2366 pmap_invalidate_page(pv->pv_va); 2367 rv = true; 2368 } 2369 pmap_switch(oldpmap); 2370 PMAP_UNLOCK(pmap); 2371 } 2372 2373 rw_exit(&pvh_global_lock); 2374 2375 return (rv); 2376 } 2377 2378 /* 2379 * pmap_clear_reference: [ INTERFACE ] 2380 * 2381 * Clear the reference bit on the specified physical page. 2382 * 2383 * returns true or false 2384 * indicating whether or not the ``referenced'' attribute was 2385 * set on the page before it was cleared 2386 */ 2387 bool 2388 pmap_clear_reference(struct vm_page *pg) 2389 { 2390 struct ia64_lpte *pte; 2391 pmap_t oldpmap, pmap; 2392 pv_entry_t pv; 2393 bool rv; 2394 2395 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2396 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0); 2397 2398 KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed", 2399 __func__, pg); 2400 2401 rv = false; 2402 2403 rw_enter(&pvh_global_lock, RW_WRITER); 2404 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) { 2405 pmap = PV_PMAP(pv); 2406 PMAP_LOCK(pmap); 2407 oldpmap = pmap_switch(pmap); 2408 pte = pmap_find_vhpt(pv->pv_va); 2409 KASSERTMSG(pte != NULL, "pte"); 2410 if (pmap_accessed(pte)) { 2411 rv = true; 2412 pmap_clear_accessed(pte); 2413 pmap_invalidate_page(pv->pv_va); 2414 } 2415 pmap_switch(oldpmap); 2416 PMAP_UNLOCK(pmap); 2417 } 2418 2419 rw_exit(&pvh_global_lock); 2420 return (rv); 2421 } 2422 2423 /* 2424 * pmap_is_modified: [ INTERFACE ] 2425 * 2426 */ 2427 bool 2428 pmap_is_modified(struct vm_page *pg) 2429 { 2430 struct ia64_lpte *pte; 2431 pmap_t oldpmap, pmap; 2432 pv_entry_t pv; 2433 bool rv; 2434 2435 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2436 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0); 2437 2438 KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed", 2439 __func__, pg); 2440 rv = false; 2441 2442 /* 2443 * If the page is not exclusive busied, then PGA_WRITEABLE cannot be 2444 * concurrently set while the object is locked. Thus, if PGA_WRITEABLE 2445 * is clear, no PTEs can be dirty. 2446 */ 2447 /* XXX freebsd 2448 VM_OBJECT_ASSERT_WLOCKED(m->object); 2449 if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) 2450 return (rv); 2451 */ 2452 rw_enter(&pvh_global_lock, RW_WRITER); 2453 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) { 2454 pmap = PV_PMAP(pv); 2455 PMAP_LOCK(pmap); 2456 oldpmap = pmap_switch(pmap); 2457 pte = pmap_find_vhpt(pv->pv_va); 2458 pmap_switch(oldpmap); 2459 KASSERTMSG(pte != NULL, "pte"); 2460 2461 rv = pmap_dirty(pte) ? true : false; 2462 PMAP_UNLOCK(pmap); 2463 if (rv) 2464 break; 2465 } 2466 2467 rw_exit(&pvh_global_lock); 2468 return (rv); 2469 } 2470 2471 /* 2472 * pmap_is_referenced: [ INTERFACE ] 2473 * 2474 * Test whether or not the ``referenced'' attribute is set on 2475 * page pg. 2476 * 2477 */ 2478 bool 2479 pmap_is_referenced(struct vm_page *pg) 2480 { 2481 struct ia64_lpte *pte; 2482 pmap_t oldpmap, pmap; 2483 pv_entry_t pv; 2484 bool rv; 2485 2486 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2487 UVMHIST_LOG(maphist, "(m=%p)", pg, 0, 0, 0); 2488 2489 KASSERTMSG(vm_page_is_managed(pg), "%s: page %p is not managed", 2490 __func__, pg); 2491 2492 rv = false; 2493 rw_enter(&pvh_global_lock, RW_WRITER); 2494 TAILQ_FOREACH(pv, &pg->mdpage.pv_list, pv_list) { 2495 pmap = PV_PMAP(pv); 2496 PMAP_LOCK(pmap); 2497 oldpmap = pmap_switch(pmap); 2498 pte = pmap_find_vhpt(pv->pv_va); 2499 KASSERTMSG(pte != NULL, "pte"); 2500 rv = pmap_accessed(pte) ? true : false; 2501 pmap_switch(oldpmap); 2502 PMAP_UNLOCK(pmap); 2503 if (rv) 2504 break; 2505 } 2506 2507 rw_exit(&pvh_global_lock); 2508 return (rv); 2509 } 2510 2511 /* 2512 * pmap_phys_address: [ INTERFACE ] 2513 * 2514 * Return the physical address corresponding to the specified 2515 * cookie. Used by the device pager to decode a device driver's 2516 * mmap entry point return value. 2517 * 2518 * Note: no locking is necessary in this function. 2519 */ 2520 paddr_t 2521 pmap_phys_address(paddr_t ppn) 2522 { 2523 return ia64_ptob(ppn); 2524 } 2525 2526 pmap_t 2527 pmap_switch(pmap_t pm) 2528 { 2529 pmap_t prevpm; 2530 int i; 2531 2532 critical_enter(); 2533 2534 prevpm = curcpu()->ci_pmap; 2535 if (prevpm == pm) 2536 goto out; 2537 if (pm == NULL) { 2538 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) { 2539 ia64_set_rr(IA64_RR_BASE(i), 2540 (i << 8)|(PAGE_SHIFT << 2)|1); 2541 } 2542 } else { 2543 for (i = 0; i < IA64_VM_MINKERN_REGION; i++) { 2544 ia64_set_rr(IA64_RR_BASE(i), 2545 (pm->pm_rid[i] << 8)|(PAGE_SHIFT << 2)|1); 2546 } 2547 } 2548 2549 /* XXX */ 2550 ia64_srlz_d(); 2551 curcpu()->ci_pmap = pm; 2552 ia64_srlz_d(); 2553 2554 out: 2555 critical_exit(); 2556 return (prevpm); 2557 } 2558 2559 2560 /* 2561 * Synchronize CPU instruction caches of the specified range. 2562 * Same as freebsd 2563 * void pmap_sync_icache(pmap_t pm, vaddr_t va, vsize_t sz) 2564 */ 2565 void 2566 pmap_procwr(struct proc *p, vaddr_t va, vsize_t sz) 2567 { 2568 pmap_t oldpm; 2569 struct ia64_lpte *pte; 2570 vaddr_t lim; 2571 vsize_t len; 2572 2573 struct pmap * const pm = p->p_vmspace->vm_map.pmap; 2574 2575 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2576 UVMHIST_LOG(maphist, "(pm=%p, va=%#lx, sz=%#lx)", pm, va, sz, 0); 2577 2578 sz += va & 31; 2579 va &= ~31; 2580 sz = (sz + 31) & ~31; 2581 2582 PMAP_LOCK(pm); 2583 oldpm = pmap_switch(pm); 2584 while (sz > 0) { 2585 lim = round_page(va); 2586 len = MIN(lim - va, sz); 2587 pte = pmap_find_vhpt(va); 2588 if (pte != NULL && pmap_present(pte)) 2589 ia64_sync_icache(va, len); 2590 va += len; 2591 sz -= len; 2592 } 2593 pmap_switch(oldpm); 2594 PMAP_UNLOCK(pm); 2595 } 2596 2597 /* 2598 * Routine: pmap_remove_all_phys 2599 * Function: 2600 * Removes this physical page from 2601 * all physical maps in which it resides. 2602 * Reflects back modify bits to the pager. 2603 * 2604 * Notes: 2605 * Original versions of this routine were very 2606 * inefficient because they iteratively called 2607 * pmap_remove (slow...) 2608 */ 2609 void 2610 pmap_remove_all_phys(struct vm_page *m) 2611 { 2612 pmap_t oldpmap; 2613 pv_entry_t pv; 2614 2615 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2616 UVMHIST_LOG(maphist, "(m=%p)", m, 0, 0, 0); 2617 2618 KASSERTMSG(vm_page_is_managed(m), "%s: page %p is not managed", 2619 __func__, m); 2620 2621 rw_enter(&pvh_global_lock, RW_WRITER); 2622 while ((pv = TAILQ_FIRST(&m->mdpage.pv_list)) != NULL) { 2623 struct ia64_lpte *pte; 2624 pmap_t pmap = PV_PMAP(pv); 2625 vaddr_t va = pv->pv_va; 2626 2627 PMAP_LOCK(pmap); 2628 oldpmap = pmap_switch(pmap); 2629 pte = pmap_find_vhpt(va); 2630 KASSERTMSG(pte != NULL, "pte"); 2631 if (pmap_ppn(pte) != VM_PAGE_TO_PHYS(m)) 2632 panic("%s: pv_table for %lx is inconsistent", 2633 __func__, VM_PAGE_TO_PHYS(m)); 2634 pmap_remove_pte(pmap, pte, va, pv, 1); 2635 pmap_switch(oldpmap); 2636 PMAP_UNLOCK(pmap); 2637 } 2638 /* XXX freebsd 2639 vm_page_aflag_clear(m, PGA_WRITEABLE); 2640 */ 2641 m->flags |= PG_RDONLY; 2642 2643 rw_exit(&pvh_global_lock); 2644 } 2645 2646 /* 2647 * vm_page_alloc1: 2648 * 2649 * Allocate and return a memory cell with no associated object. 2650 */ 2651 static struct vm_page 2652 *vm_page_alloc1(void) 2653 { 2654 struct vm_page *pg; 2655 2656 pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_USERESERVE|UVM_PGA_ZERO); 2657 if (pg) { 2658 pg->wire_count = 1; /* no mappings yet */ 2659 pg->flags &= ~PG_BUSY; /* never busy */ 2660 } 2661 return pg; 2662 } 2663 2664 /* 2665 * vm_page_free1: 2666 * 2667 * Returns the given page to the free list, 2668 * disassociating it with any VM object. 2669 * 2670 * Object and page must be locked prior to entry. 2671 */ 2672 static void 2673 vm_page_free1(struct vm_page *pg) 2674 { 2675 KASSERT(pg->flags != (PG_CLEAN|PG_FAKE)); /* Freeing invalid page */ 2676 2677 pg->flags |= PG_BUSY; 2678 pg->wire_count = 0; 2679 uvm_pagefree(pg); 2680 } 2681 2682 /* 2683 * pmap_flag_to_attr 2684 * 2685 * Convert pmap_enter/pmap_kenter_pa flags to memory attributes 2686 */ 2687 static vm_memattr_t 2688 pmap_flags_to_memattr(u_int flags) 2689 { 2690 u_int cacheflags = flags & PMAP_CACHE_MASK; 2691 2692 #if 0 2693 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2694 UVMHIST_LOG(maphist, "(PMAP_NOCACHE=%u, PMAP_WRITE_COMBINE=%u, " 2695 "PMAP_WRITE_BACK=%u, PMAP_NOCACHE_OVR=%u)", 2696 (flags & PMAP_NOCACHE) != 0, (flags & PMAP_WRITE_COMBINE) != 0, 2697 (flags & PMAP_WRITE_BACK) != 0, (flags & PMAP_NOCACHE_OVR) != 0); 2698 #endif 2699 switch (cacheflags) { 2700 case PMAP_NOCACHE: 2701 return VM_MEMATTR_UNCACHEABLE; 2702 case PMAP_WRITE_COMBINE: 2703 /* XXX implement if possible */ 2704 KASSERT(1); 2705 return VM_MEMATTR_WRITE_COMBINING; 2706 case PMAP_WRITE_BACK: 2707 return VM_MEMATTR_WRITE_BACK; 2708 case PMAP_NOCACHE_OVR: 2709 default: 2710 return VM_MEMATTR_DEFAULT; 2711 } 2712 } 2713 2714 #ifdef DEBUG 2715 /* 2716 * Test ref/modify handling. 2717 */ 2718 2719 static void 2720 pmap_testout(void) 2721 { 2722 vaddr_t va; 2723 volatile int *loc; 2724 int val = 0; 2725 paddr_t pa; 2726 struct vm_page *pg; 2727 int ref, mod; 2728 bool extracted; 2729 2730 /* Allocate a page */ 2731 va = (vaddr_t)uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_WIRED | UVM_KMF_ZERO); 2732 KASSERT(va != 0); 2733 loc = (int*)va; 2734 2735 extracted = pmap_extract(pmap_kernel(), va, &pa); 2736 printf("va %p pa %lx extracted %u\n", (void *)(u_long)va, pa, extracted); 2737 printf("kextract %lx\n", pmap_kextract(va)); 2738 2739 pg = PHYS_TO_VM_PAGE(pa); 2740 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2741 pmap_update(pmap_kernel()); 2742 2743 ref = pmap_is_referenced(pg); 2744 mod = pmap_is_modified(pg); 2745 2746 printf("Entered page va %p pa %lx: ref %d, mod %d\n", 2747 (void *)(u_long)va, (long)pa, ref, mod); 2748 2749 /* Now clear reference and modify */ 2750 ref = pmap_clear_reference(pg); 2751 mod = pmap_clear_modify(pg); 2752 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2753 (void *)(u_long)va, (long)pa, 2754 ref, mod); 2755 2756 /* Check it's properly cleared */ 2757 ref = pmap_is_referenced(pg); 2758 mod = pmap_is_modified(pg); 2759 printf("Checking cleared page: ref %d, mod %d\n", 2760 ref, mod); 2761 2762 /* Reference page */ 2763 val = *loc; 2764 2765 ref = pmap_is_referenced(pg); 2766 mod = pmap_is_modified(pg); 2767 printf("Referenced page: ref %d, mod %d val %x\n", 2768 ref, mod, val); 2769 2770 /* Now clear reference and modify */ 2771 ref = pmap_clear_reference(pg); 2772 mod = pmap_clear_modify(pg); 2773 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2774 (void *)(u_long)va, (long)pa, 2775 ref, mod); 2776 2777 /* Modify page */ 2778 *loc = 1; 2779 2780 ref = pmap_is_referenced(pg); 2781 mod = pmap_is_modified(pg); 2782 printf("Modified page: ref %d, mod %d\n", 2783 ref, mod); 2784 2785 /* Now clear reference and modify */ 2786 ref = pmap_clear_reference(pg); 2787 mod = pmap_clear_modify(pg); 2788 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2789 (void *)(u_long)va, (long)pa, 2790 ref, mod); 2791 2792 /* Check it's properly cleared */ 2793 ref = pmap_is_referenced(pg); 2794 mod = pmap_is_modified(pg); 2795 printf("Checking cleared page: ref %d, mod %d\n", 2796 ref, mod); 2797 2798 /* Modify page */ 2799 *loc = 1; 2800 2801 ref = pmap_is_referenced(pg); 2802 mod = pmap_is_modified(pg); 2803 printf("Modified page: ref %d, mod %d\n", 2804 ref, mod); 2805 2806 /* Check pmap_protect() */ 2807 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_READ); 2808 pmap_update(pmap_kernel()); 2809 ref = pmap_is_referenced(pg); 2810 mod = pmap_is_modified(pg); 2811 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 2812 ref, mod); 2813 2814 /* Now clear reference and modify */ 2815 ref = pmap_clear_reference(pg); 2816 mod = pmap_clear_modify(pg); 2817 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2818 (void *)(u_long)va, (long)pa, 2819 ref, mod); 2820 2821 /* Modify page */ 2822 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2823 pmap_update(pmap_kernel()); 2824 *loc = 1; 2825 2826 ref = pmap_is_referenced(pg); 2827 mod = pmap_is_modified(pg); 2828 printf("Modified page: ref %d, mod %d\n", 2829 ref, mod); 2830 2831 /* Check pmap_protect() */ 2832 pmap_protect(pmap_kernel(), va, va+1, VM_PROT_NONE); 2833 pmap_update(pmap_kernel()); 2834 ref = pmap_is_referenced(pg); 2835 mod = pmap_is_modified(pg); 2836 printf("pmap_protect(VM_PROT_READ): ref %d, mod %d\n", 2837 ref, mod); 2838 2839 /* Now clear reference and modify */ 2840 ref = pmap_clear_reference(pg); 2841 mod = pmap_clear_modify(pg); 2842 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2843 (void *)(u_long)va, (long)pa, 2844 ref, mod); 2845 2846 /* Modify page */ 2847 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2848 pmap_update(pmap_kernel()); 2849 *loc = 1; 2850 2851 ref = pmap_is_referenced(pg); 2852 mod = pmap_is_modified(pg); 2853 printf("Modified page: ref %d, mod %d\n", 2854 ref, mod); 2855 2856 /* Check pmap_pag_protect() */ 2857 pmap_page_protect(pg, VM_PROT_READ); 2858 ref = pmap_is_referenced(pg); 2859 mod = pmap_is_modified(pg); 2860 printf("pmap_protect(): ref %d, mod %d\n", 2861 ref, mod); 2862 2863 /* Now clear reference and modify */ 2864 ref = pmap_clear_reference(pg); 2865 mod = pmap_clear_modify(pg); 2866 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2867 (void *)(u_long)va, (long)pa, 2868 ref, mod); 2869 2870 2871 /* Modify page */ 2872 pmap_enter(pmap_kernel(), va, pa, VM_PROT_ALL, 0); 2873 pmap_update(pmap_kernel()); 2874 *loc = 1; 2875 2876 ref = pmap_is_referenced(pg); 2877 mod = pmap_is_modified(pg); 2878 printf("Modified page: ref %d, mod %d\n", 2879 ref, mod); 2880 2881 /* Check pmap_pag_protect() */ 2882 pmap_page_protect(pg, VM_PROT_NONE); 2883 ref = pmap_is_referenced(pg); 2884 mod = pmap_is_modified(pg); 2885 printf("pmap_protect(): ref %d, mod %d\n", 2886 ref, mod); 2887 2888 /* Now clear reference and modify */ 2889 ref = pmap_clear_reference(pg); 2890 mod = pmap_clear_modify(pg); 2891 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2892 (void *)(u_long)va, (long)pa, 2893 ref, mod); 2894 2895 /* Unmap page */ 2896 pmap_remove(pmap_kernel(), va, va+1); 2897 pmap_update(pmap_kernel()); 2898 ref = pmap_is_referenced(pg); 2899 mod = pmap_is_modified(pg); 2900 printf("Unmapped page: ref %d, mod %d\n", ref, mod); 2901 2902 /* Now clear reference and modify */ 2903 ref = pmap_clear_reference(pg); 2904 mod = pmap_clear_modify(pg); 2905 printf("Clearing page va %p pa %lx: ref %d, mod %d\n", 2906 (void *)(u_long)va, (long)pa, ref, mod); 2907 2908 /* Check it's properly cleared */ 2909 ref = pmap_is_referenced(pg); 2910 mod = pmap_is_modified(pg); 2911 printf("Checking cleared page: ref %d, mod %d\n", 2912 ref, mod); 2913 2914 pmap_remove(pmap_kernel(), va, va+1); 2915 pmap_update(pmap_kernel()); 2916 //pmap_free_page(pa, cpus_active); 2917 uvm_km_free(kernel_map, (vaddr_t)va, PAGE_SIZE, 2918 UVM_KMF_WIRED | UVM_KMF_ZERO); 2919 2920 panic("end of testout"); 2921 } 2922 #endif 2923
Indexes created Sun Sep 21 19:09:51 GMT 2025