1 /* $NetBSD: pmap.c,v 1.122 2023/08/02 09:18:14 macallan Exp $ */ 2 3 /*- 4 * Copyright (c) 2001, 2002, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matthew Fredette. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* $OpenBSD: pmap.c,v 1.132 2008/04/18 06:42:21 djm Exp $ */ 33 34 /* 35 * Copyright (c) 1998-2004 Michael Shalayeff 36 * All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 48 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 49 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 50 * IN NO EVENT SHALL THE AUTHOR OR HIS RELATIVES BE LIABLE FOR ANY DIRECT, 51 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 52 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 53 * SERVICES; LOSS OF MIND, USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 55 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 56 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 57 * THE POSSIBILITY OF SUCH DAMAGE. 58 */ 59 /* 60 * References: 61 * 1. PA7100LC ERS, Hewlett-Packard, March 30 1999, Public version 1.0 62 * 2. PA7300LC ERS, Hewlett-Packard, March 18 1996, Version 1.0 63 * 3. PA-RISC 1.1 Architecture and Instruction Set Reference Manual, 64 * Hewlett-Packard, February 1994, Third Edition 65 */ 66 67 #include <sys/cdefs.h> 68 __KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.122 2023/08/02 09:18:14 macallan Exp $"); 69 70 #include "opt_cputype.h" 71 72 #include <sys/param.h> 73 #include <sys/atomic.h> 74 #include <sys/mutex.h> 75 #include <sys/pool.h> 76 #include <sys/proc.h> 77 #include <sys/rwlock.h> 78 #include <sys/systm.h> 79 80 #include <uvm/uvm.h> 81 #include <uvm/uvm_page_array.h> 82 83 #include <machine/cpu.h> 84 #include <machine/cpufunc.h> 85 #include <machine/iomod.h> 86 #include <machine/pcb.h> 87 #include <machine/pmap.h> 88 #include <machine/psl.h> 89 #include <machine/pte.h> 90 #include <machine/reg.h> 91 92 #include <hppa/hppa/hpt.h> 93 #include <hppa/hppa/machdep.h> 94 95 #if defined(DDB) 96 #include <ddb/db_output.h> 97 #endif 98 99 int pmap_hptsize = 16 * PAGE_SIZE; /* patchable */ 100 vaddr_t pmap_hpt; 101 102 static struct pmap kernel_pmap_store; 103 struct pmap *const kernel_pmap_ptr = &kernel_pmap_store; 104 105 int hppa_sid_max = HPPA_SID_MAX; 106 struct pool pmap_pool; 107 struct pool pmap_pv_pool; 108 int pmap_pvlowat = 252; 109 bool pmap_initialized = false; 110 111 static kmutex_t pmaps_lock; 112 113 static union pmap_pv_locks { 114 kmutex_t lock; 115 char padding[COHERENCY_UNIT]; 116 } pmap_pv_locks[64] __aligned(COHERENCY_UNIT); 117 118 #define PMAP_PV_LOCK(md) \ 119 ((uintptr_t)(md) >> 7 & (__arraycount(pmap_pv_locks) - 1)) 120 121 u_int hppa_prot[8]; 122 u_int sid_counter; 123 124 static const struct uvm_pagerops pmap_pager = { 125 /* nothing */ 126 }; 127 128 /* 129 * Page 3-6 of the "PA-RISC 1.1 Architecture and Instruction Set 130 * Reference Manual" (HP part number 09740-90039) defines equivalent 131 * and non-equivalent virtual addresses in the cache. 132 * 133 * This macro evaluates to true iff the two space/virtual address 134 * combinations are non-equivalent aliases, and therefore will find 135 * two different locations in the cache. 136 * 137 * NB: currently, the CPU-specific desidhash() functions disable the 138 * use of the space in all cache hashing functions. This means that 139 * this macro definition is stricter than it has to be (because it 140 * takes space into account), but one day cache space hashing should 141 * be re-enabled. Cache space hashing should yield better performance 142 * through better utilization of the cache, assuming that most aliasing 143 * is the read-only kind, which we do allow in the cache. 144 */ 145 #define NON_EQUIVALENT_ALIAS(sp1, va1, sp2, va2) \ 146 (((((va1) ^ (va2)) & ~HPPA_PGAMASK) != 0) || \ 147 ((((sp1) ^ (sp2)) & ~HPPA_SPAMASK) != 0)) 148 149 /* Prototypes. */ 150 struct vm_page *pmap_pagealloc(struct uvm_object *, voff_t); 151 void pmap_pagefree(struct vm_page *); 152 153 static inline void pmap_lock(struct pmap *); 154 static inline void pmap_unlock(struct pmap *); 155 static inline bool pmap_trylock(struct pmap *); 156 157 static inline void pmap_sdir_set(pa_space_t, volatile uint32_t *); 158 static inline uint32_t *pmap_sdir_get(pa_space_t); 159 160 static inline volatile pt_entry_t *pmap_pde_get(volatile uint32_t *, vaddr_t); 161 static inline void pmap_pde_set(pmap_t, vaddr_t, paddr_t); 162 static inline pt_entry_t *pmap_pde_alloc(pmap_t, vaddr_t, struct vm_page **); 163 static inline struct vm_page *pmap_pde_ptp(pmap_t, volatile pt_entry_t *); 164 static inline void pmap_pde_release(pmap_t, vaddr_t, struct vm_page *); 165 166 static inline volatile pt_entry_t *pmap_pde_get(volatile uint32_t *, vaddr_t); 167 static inline void pmap_pde_set(pmap_t, vaddr_t, paddr_t); 168 169 void pmap_pte_flush(pmap_t, vaddr_t, pt_entry_t); 170 171 static inline pt_entry_t pmap_pte_get(volatile pt_entry_t *, vaddr_t); 172 static inline void pmap_pte_set(volatile pt_entry_t *, vaddr_t, pt_entry_t); 173 174 static inline pt_entry_t pmap_vp_find(pmap_t, vaddr_t); 175 176 static inline struct pv_entry *pmap_pv_alloc(void); 177 static inline void pmap_pv_free(struct pv_entry *); 178 static inline void pmap_pv_enter(struct vm_page *, struct pv_entry *, pmap_t, 179 vaddr_t , struct vm_page *, u_int); 180 static inline struct pv_entry *pmap_pv_remove(struct vm_page *, pmap_t, 181 vaddr_t); 182 static inline void pmap_pv_lock(const struct vm_page_md *md); 183 static inline void pmap_pv_unlock(const struct vm_page_md *md); 184 static inline bool pmap_pv_locked(const struct vm_page_md *md); 185 186 static inline void pmap_flush_page(struct vm_page *, bool); 187 static void pmap_resolve_alias(struct vm_page *, struct pmap *, vaddr_t, 188 pt_entry_t); 189 static void pmap_syncicache_page(struct vm_page *, pmap_t, vaddr_t); 190 191 static void pmap_page_physload(paddr_t, paddr_t); 192 193 void pmap_copy_page(paddr_t, paddr_t); 194 195 #ifdef USE_HPT 196 static inline struct hpt_entry *pmap_hash(pmap_t, vaddr_t); 197 static inline uint32_t pmap_vtag(pmap_t, vaddr_t); 198 199 #ifdef DDB 200 void pmap_hptdump(void); 201 #endif 202 #endif 203 204 #ifdef DDB 205 void pmap_dump_table(pa_space_t, vaddr_t); 206 void pmap_dump_pv(paddr_t); 207 #endif 208 209 #define IS_IOPAGE_P(pa) ((pa) >= HPPA_IOBEGIN) 210 #define IS_PVFEXEC_P(f) (((f) & PVF_EXEC) != 0) 211 212 /* un-invert PVF_REF */ 213 #define pmap_pvh_attrs(a) \ 214 (((a) & (PVF_MOD|PVF_REF)) ^ PVF_REF) 215 216 static inline void 217 pmap_lock(struct pmap *pm) 218 { 219 220 rw_enter(pm->pm_lock, RW_WRITER); 221 } 222 223 static inline void 224 pmap_unlock(struct pmap *pm) 225 { 226 227 rw_exit(pm->pm_lock); 228 } 229 230 static inline bool 231 pmap_trylock(struct pmap *pm) 232 { 233 234 return rw_tryenter(pm->pm_lock, RW_WRITER); 235 } 236 237 static inline void 238 pmap_pv_lock(const struct vm_page_md *md) 239 { 240 241 mutex_enter(&pmap_pv_locks[PMAP_PV_LOCK(md)].lock); 242 } 243 244 static inline void 245 pmap_pv_unlock(const struct vm_page_md *md) 246 { 247 248 mutex_exit(&pmap_pv_locks[PMAP_PV_LOCK(md)].lock); 249 } 250 251 static inline bool 252 pmap_pv_locked(const struct vm_page_md *md) 253 { 254 255 return mutex_owned(&pmap_pv_locks[PMAP_PV_LOCK(md)].lock); 256 } 257 258 struct vm_page * 259 pmap_pagealloc(struct uvm_object *obj, voff_t off) 260 { 261 struct vm_page *pg; 262 263 if ((pg = uvm_pagealloc(obj, off, NULL, 264 UVM_PGA_USERESERVE | UVM_PGA_ZERO)) == NULL) 265 printf("pmap_pagealloc fail\n"); 266 267 return (pg); 268 } 269 270 void 271 pmap_pagefree(struct vm_page *pg) 272 { 273 paddr_t pa = VM_PAGE_TO_PHYS(pg); 274 pdcache(HPPA_SID_KERNEL, pa, PAGE_SIZE); 275 276 #if defined(HP8000_CPU) || defined(HP8200_CPU) || \ 277 defined(HP8500_CPU) || defined(HP8600_CPU) 278 pdtlb(HPPA_SID_KERNEL, pa); 279 pitlb(HPPA_SID_KERNEL, pa); 280 #endif 281 uvm_pagefree(pg); 282 } 283 284 #ifdef USE_HPT 285 /* 286 * This hash function is the one used by the hardware TLB walker on the 7100LC. 287 */ 288 static inline struct hpt_entry * 289 pmap_hash(pmap_t pmap, vaddr_t va) 290 { 291 292 return (struct hpt_entry *)(pmap_hpt + 293 (((va >> 8) ^ (pmap->pm_space << 9)) & (pmap_hptsize - 1))); 294 } 295 296 static inline uint32_t 297 pmap_vtag(pmap_t pmap, vaddr_t va) 298 { 299 300 return (0x80000000 | (pmap->pm_space & 0xffff) | 301 ((va >> 1) & 0x7fff0000)); 302 } 303 #endif 304 305 static inline void 306 pmap_sdir_set(pa_space_t space, volatile uint32_t *pd) 307 { 308 volatile uint32_t *vtop; 309 310 mfctl(CR_VTOP, vtop); 311 312 KASSERT(vtop != NULL); 313 314 vtop[space] = (uint32_t)pd; 315 } 316 317 static inline uint32_t * 318 pmap_sdir_get(pa_space_t space) 319 { 320 uint32_t *vtop; 321 322 mfctl(CR_VTOP, vtop); 323 return ((uint32_t *)vtop[space]); 324 } 325 326 static inline volatile pt_entry_t * 327 pmap_pde_get(volatile uint32_t *pd, vaddr_t va) 328 { 329 330 return ((pt_entry_t *)pd[va >> 22]); 331 } 332 333 static inline void 334 pmap_pde_set(pmap_t pm, vaddr_t va, paddr_t ptp) 335 { 336 UVMHIST_FUNC(__func__); 337 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx ptp %#jx", (uintptr_t)pm, 338 va, ptp, 0); 339 340 KASSERT((ptp & PGOFSET) == 0); 341 342 pm->pm_pdir[va >> 22] = ptp; 343 } 344 345 static inline pt_entry_t * 346 pmap_pde_alloc(pmap_t pm, vaddr_t va, struct vm_page **pdep) 347 { 348 struct vm_page *pg; 349 paddr_t pa; 350 351 UVMHIST_FUNC(__func__); 352 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pdep %#jx", (uintptr_t)pm, 353 va, (uintptr_t)pdep, 0); 354 355 356 KASSERT(pm != pmap_kernel()); 357 KASSERT(rw_write_held(pm->pm_lock)); 358 359 pg = pmap_pagealloc(&pm->pm_obj, va); 360 361 if (pg == NULL) 362 return NULL; 363 364 pa = VM_PAGE_TO_PHYS(pg); 365 366 UVMHIST_LOG(maphist, "pde %#jx", pa, 0, 0, 0); 367 368 pg->flags &= ~PG_BUSY; /* never busy */ 369 pg->wire_count = 1; /* no mappings yet */ 370 pmap_pde_set(pm, va, pa); 371 pm->pm_stats.resident_count++; /* count PTP as resident */ 372 pm->pm_ptphint = pg; 373 if (pdep) 374 *pdep = pg; 375 return ((pt_entry_t *)pa); 376 } 377 378 static inline struct vm_page * 379 pmap_pde_ptp(pmap_t pm, volatile pt_entry_t *pde) 380 { 381 paddr_t pa = (paddr_t)pde; 382 383 UVMHIST_FUNC(__func__); 384 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pdep %#jx", (uintptr_t)pm, 385 (uintptr_t)pde, 0, 0); 386 387 if (pm->pm_ptphint && VM_PAGE_TO_PHYS(pm->pm_ptphint) == pa) 388 return (pm->pm_ptphint); 389 390 UVMHIST_LOG(maphist, "<--- done (%#jx)", 391 (uintptr_t)PHYS_TO_VM_PAGE(pa), 0, 0, 0); 392 393 return (PHYS_TO_VM_PAGE(pa)); 394 } 395 396 static inline void 397 pmap_pde_release(pmap_t pmap, vaddr_t va, struct vm_page *ptp) 398 { 399 UVMHIST_FUNC(__func__); 400 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx ptp %#jx", (uintptr_t)pmap, 401 va, (uintptr_t)ptp, 0); 402 403 KASSERT(pmap != pmap_kernel()); 404 if (--ptp->wire_count <= 1) { 405 UVMHIST_LOG(maphist, "disposing ptp %#jx", (uintptr_t)ptp, 0, 406 0, 0); 407 pmap_pde_set(pmap, va, 0); 408 pmap->pm_stats.resident_count--; 409 if (pmap->pm_ptphint == ptp) 410 pmap->pm_ptphint = NULL; 411 ptp->wire_count = 0; 412 413 KASSERT((ptp->flags & PG_BUSY) == 0); 414 415 pmap_pagefree(ptp); 416 } 417 } 418 419 static inline pt_entry_t 420 pmap_pte_get(volatile pt_entry_t *pde, vaddr_t va) 421 { 422 423 return (pde[(va >> 12) & 0x3ff]); 424 } 425 426 static inline void 427 pmap_pte_set(volatile pt_entry_t *pde, vaddr_t va, pt_entry_t pte) 428 { 429 430 /* too verbose due to hppa_pagezero_{,un}map */ 431 #if 0 432 UVMHIST_FUNC(__func__); 433 UVMHIST_CALLARGS(maphist, "pdep %#jx va %#jx pte %#jx", (uintptr_t)pde, 434 va, pte, 0); 435 #endif 436 437 KASSERT(pde != NULL); 438 KASSERT(((paddr_t)pde & PGOFSET) == 0); 439 440 pde[(va >> 12) & 0x3ff] = pte; 441 } 442 443 void 444 pmap_pte_flush(pmap_t pmap, vaddr_t va, pt_entry_t pte) 445 { 446 447 UVMHIST_FUNC(__func__); 448 if (pmap != pmap_kernel() && va != 0) { 449 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pte %#jx", 450 (uintptr_t)pmap, va, (uintptr_t)pte, 0); 451 } 452 453 fdcache(pmap->pm_space, va, PAGE_SIZE); 454 if (pte & PTE_PROT(TLB_EXECUTE)) { 455 ficache(pmap->pm_space, va, PAGE_SIZE); 456 pitlb(pmap->pm_space, va); 457 } 458 pdtlb(pmap->pm_space, va); 459 #ifdef USE_HPT 460 if (pmap_hpt) { 461 struct hpt_entry *hpt; 462 hpt = pmap_hash(pmap, va); 463 if (hpt->hpt_valid && 464 hpt->hpt_space == pmap->pm_space && 465 hpt->hpt_vpn == ((va >> 1) & 0x7fff0000)) 466 hpt->hpt_space = 0xffff; 467 } 468 #endif 469 } 470 471 static inline pt_entry_t 472 pmap_vp_find(pmap_t pm, vaddr_t va) 473 { 474 volatile pt_entry_t *pde; 475 476 if (!(pde = pmap_pde_get(pm->pm_pdir, va))) 477 return (0); 478 479 return (pmap_pte_get(pde, va)); 480 } 481 482 #ifdef DDB 483 void 484 pmap_dump_table(pa_space_t space, vaddr_t sva) 485 { 486 char buf[64]; 487 volatile pt_entry_t *pde = NULL; 488 vaddr_t va = sva; 489 vaddr_t pdemask = 1; 490 pt_entry_t pte; 491 uint32_t *pd; 492 493 if (space > hppa_sid_max) 494 return; 495 496 pd = pmap_sdir_get(space); 497 if (!pd) 498 return; 499 500 do { 501 if (pdemask != (va & PDE_MASK)) { 502 pdemask = va & PDE_MASK; 503 pde = pmap_pde_get(pd, va); 504 if (!pde) { 505 va = pdemask + PDE_SIZE; 506 continue; 507 } 508 db_printf("%x:%8p:\n", space, pde); 509 } 510 511 pte = pmap_pte_get(pde, va); 512 if (pte) { 513 snprintb(buf, sizeof(buf), TLB_BITS, 514 TLB_PROT(pte & PAGE_MASK)); 515 db_printf("0x%08lx-0x%08x:%s\n", va, pte & ~PAGE_MASK, 516 buf); 517 } 518 va += PAGE_SIZE; 519 } while (va != 0); 520 } 521 522 void 523 pmap_dump_pv(paddr_t pa) 524 { 525 struct vm_page *pg; 526 struct vm_page_md *md; 527 struct pv_entry *pve; 528 529 pg = PHYS_TO_VM_PAGE(pa); 530 if (pg == NULL) 531 return; 532 533 md = VM_PAGE_TO_MD(pg); 534 db_printf("pg %p attr 0x%08x\n", pg, md->pvh_attrs); 535 for (pve = md->pvh_list; pve; pve = pve->pv_next) 536 db_printf("%x:%lx\n", pve->pv_pmap->pm_space, 537 pve->pv_va & PV_VAMASK); 538 } 539 #endif 540 541 static void 542 pmap_resolve_alias(struct vm_page *pg, struct pmap *pm, vaddr_t va, 543 pt_entry_t pte) 544 { 545 546 UVMHIST_FUNC(__func__); 547 UVMHIST_CALLARGS(maphist, "pg %#jx pm %#jx va %#jx pte %#jx", 548 (uintptr_t)pg, (uintptr_t)pm, va, pte); 549 550 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 551 struct pv_entry *pve, *npve, **pvp; 552 553 restart: 554 pmap_pv_lock(md); 555 pvp = &md->pvh_list; 556 for (pve = md->pvh_list; pve; pve = npve) { 557 const pmap_t ppm = pve->pv_pmap; 558 const vaddr_t pva = pve->pv_va & PV_VAMASK; 559 560 UVMHIST_LOG(maphist, "... pm %#jx va %#jx", (uintptr_t)ppm, 561 pva, 0, 0); 562 563 npve = pve->pv_next; 564 565 volatile pt_entry_t *pde; 566 pt_entry_t ppte; 567 if (pve->pv_va & PV_KENTER) { 568 /* Get the pte for this mapping */ 569 pde = pmap_pde_get(ppm->pm_pdir, pva); 570 ppte = pmap_pte_get(pde, pva); 571 } else { 572 /* 573 * We have to seamlessly get a hold on the pmap's lock 574 * while holding the PV head lock, to know that the 575 * mapping is still in place and we can operate on it. 576 * If that can't be had, drop the PV head lock, wait 577 * for the pmap's lock to become available, and then 578 * try again. 579 */ 580 UVMHIST_LOG(maphist, "... pm %#jx va %#jx... checking", 581 (uintptr_t)ppm, pva, 0, 0); 582 583 bool locked = true; 584 if (pm != ppm) { 585 pmap_reference(ppm); 586 locked = pmap_trylock(ppm); 587 } 588 589 if (!locked) { 590 pmap_pv_unlock(md); 591 pmap_lock(ppm); 592 /* nothing */ 593 pmap_unlock(ppm); 594 pmap_destroy(ppm); 595 596 UVMHIST_LOG(maphist, "... failed lock", 0, 0, 0, 597 0); 598 goto restart; 599 } 600 pde = pmap_pde_get(ppm->pm_pdir, pva); 601 ppte = pmap_pte_get(pde, pva); 602 603 md->pvh_attrs |= pmap_pvh_attrs(ppte); 604 } 605 606 const bool writeable = 607 ((pte | ppte) & PTE_PROT(TLB_WRITE)) != 0; 608 609 if ((va & HPPA_PGAOFF) != (pva & HPPA_PGAOFF) && writeable) { 610 UVMHIST_LOG(maphist, 611 "aliased writeable mapping %#jx:%#jx", 612 ppm->pm_space, pva, 0, 0); 613 614 pmap_pte_flush(ppm, pva, ppte); 615 if (ppte & PTE_PROT(TLB_WIRED)) 616 ppm->pm_stats.wired_count--; 617 ppm->pm_stats.resident_count--; 618 619 if (pve->pv_va & PV_KENTER) { 620 /* 621 * This is an unmanaged mapping, it must be 622 * preserved. Move it back on the list and 623 * advance the end-of-list pointer. 624 */ 625 *pvp = pve; 626 pvp = &pve->pv_next; 627 } else { 628 pmap_pte_set(pde, pva, 0); 629 630 /* Remove pve from list */ 631 *pvp = npve; 632 633 pmap_pv_unlock(md); 634 pmap_pv_free(pve); 635 if (pm != ppm) { 636 pmap_unlock(ppm); 637 pmap_destroy(ppm); 638 639 } 640 UVMHIST_LOG(maphist, "... removed", 0, 641 0, 0, 0); 642 goto restart; 643 } 644 } else { 645 UVMHIST_LOG(maphist, "not aliased writeable mapping", 646 0,0,0,0); 647 648 if (!(pve->pv_va & PV_KENTER) && pm != ppm) { 649 pmap_unlock(ppm); 650 pmap_destroy(ppm); 651 } 652 *pvp = pve; 653 pvp = &pve->pv_next; 654 } 655 } 656 md->pvh_attrs &= ~PVF_EXEC; 657 *pvp = NULL; 658 659 #ifdef DEBUG 660 int ret = 0; 661 /* check for non-equ aliased mappings */ 662 for (pve = md->pvh_list; pve; pve = pve->pv_next) { 663 vaddr_t pva = pve->pv_va & PV_VAMASK; 664 665 UVMHIST_LOG(maphist, "... pm %#jx va %#jx", 666 (uintptr_t)pve->pv_pmap, pva, 0, 0); 667 668 pte |= pmap_vp_find(pve->pv_pmap, pva); 669 if ((va & HPPA_PGAOFF) != (pva & HPPA_PGAOFF) && 670 (pte & PTE_PROT(TLB_WRITE))) { 671 UVMHIST_LOG(maphist, 672 "aliased writable mapping %#jx:%#jx", 673 pve->pv_pmap->pm_space, pve->pv_va, 0, 0); 674 675 ret++; 676 } 677 } 678 UVMHIST_LOG(maphist, "check returned %jd", ret, 0, 0, 0); 679 #endif 680 681 pmap_pv_unlock(md); 682 683 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 684 685 return; 686 } 687 688 /* 689 * This allocates and returns a new struct pv_entry. 690 */ 691 static inline struct pv_entry * 692 pmap_pv_alloc(void) 693 { 694 struct pv_entry *pv; 695 696 pv = pool_get(&pmap_pv_pool, PR_NOWAIT); 697 698 return (pv); 699 } 700 701 static inline void 702 pmap_pv_free(struct pv_entry *pv) 703 { 704 705 if (pv->pv_ptp) 706 pmap_pde_release(pv->pv_pmap, pv->pv_va & PV_VAMASK, 707 pv->pv_ptp); 708 709 pool_put(&pmap_pv_pool, pv); 710 } 711 712 static inline void 713 pmap_pv_enter(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, 714 vaddr_t va, struct vm_page *pdep, u_int flags) 715 { 716 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 717 718 UVMHIST_FUNC(__func__); 719 UVMHIST_CALLARGS(maphist, "pg %#jx pve %#jx pm %#jx va %#jx", 720 (uintptr_t)pg, (uintptr_t)pve, (uintptr_t)pm, va); 721 UVMHIST_LOG(maphist, "...pdep %#jx flags %#jx", 722 (uintptr_t)pdep, flags, 0, 0); 723 724 KASSERT(pmap_pv_locked(md)); 725 726 pve->pv_pmap = pm; 727 pve->pv_va = va | flags; 728 pve->pv_ptp = pdep; 729 pve->pv_next = md->pvh_list; 730 md->pvh_list = pve; 731 } 732 733 static inline struct pv_entry * 734 pmap_pv_remove(struct vm_page *pg, pmap_t pmap, vaddr_t va) 735 { 736 UVMHIST_FUNC(__func__); 737 UVMHIST_CALLARGS(maphist, "pg %#jx pm %#jx va %#jx", 738 (uintptr_t)pg, (uintptr_t)pmap, va, 0); 739 740 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 741 struct pv_entry **pve, *pv; 742 743 KASSERT(pmap_pv_locked(md)); 744 745 for (pv = *(pve = &md->pvh_list); 746 pv; pv = *(pve = &(*pve)->pv_next)) { 747 if (pv->pv_pmap == pmap && (pv->pv_va & PV_VAMASK) == va) { 748 *pve = pv->pv_next; 749 break; 750 } 751 } 752 753 if (IS_PVFEXEC_P(md->pvh_attrs)) { 754 if (md->pvh_list == NULL) { 755 md->pvh_attrs &= ~PVF_EXEC; 756 } else { 757 pmap_syncicache_page(pg, pmap, va); 758 } 759 } 760 761 return (pv); 762 } 763 764 #define FIRST_16M atop(16 * 1024 * 1024) 765 766 static void 767 pmap_page_physload(paddr_t spa, paddr_t epa) 768 { 769 770 if (spa == epa) 771 return; 772 773 if (spa < FIRST_16M && epa <= FIRST_16M) { 774 uvm_page_physload(spa, epa, spa, epa, VM_FREELIST_ISADMA); 775 } else if (spa < FIRST_16M && epa > FIRST_16M) { 776 uvm_page_physload(spa, FIRST_16M, spa, FIRST_16M, 777 VM_FREELIST_ISADMA); 778 uvm_page_physload(FIRST_16M, epa, FIRST_16M, epa, 779 VM_FREELIST_DEFAULT); 780 } else { 781 uvm_page_physload(spa, epa, spa, epa, VM_FREELIST_DEFAULT); 782 } 783 784 availphysmem += epa - spa; 785 } 786 787 /* 788 * Bootstrap the system enough to run with virtual memory. 789 * Map the kernel's code, data and bss, and allocate the system page table. 790 * Called with mapping OFF. 791 * 792 * Parameters: 793 * vstart PA of first available physical page 794 */ 795 void 796 pmap_bootstrap(vaddr_t vstart) 797 { 798 UVMHIST_FUNC(__func__); 799 UVMHIST_CALLED(maphist); 800 801 vaddr_t va, addr; 802 vsize_t size; 803 extern paddr_t hppa_vtop; 804 pmap_t kpm; 805 int npdes, nkpdes; 806 extern int resvphysmem; 807 vsize_t btlb_entry_min, btlb_entry_max, btlb_entry_got; 808 paddr_t ksrx, kerx, ksro, kero, ksrw, kerw; 809 extern int usebtlb; 810 811 /* Provided by the linker script */ 812 extern int kernel_text, etext; 813 extern int __rodata_start, __rodata_end; 814 extern int __data_start; 815 816 uvm_md_init(); 817 818 hppa_prot[UVM_PROT_NONE] = TLB_AR_NA; 819 hppa_prot[UVM_PROT_READ] = TLB_AR_R; 820 hppa_prot[UVM_PROT_WRITE] = TLB_AR_RW; 821 hppa_prot[UVM_PROT_RW] = TLB_AR_RW; 822 hppa_prot[UVM_PROT_EXEC] = TLB_AR_RX; 823 hppa_prot[UVM_PROT_RX] = TLB_AR_RX; 824 hppa_prot[UVM_PROT_WX] = TLB_AR_RWX; 825 hppa_prot[UVM_PROT_RWX] = TLB_AR_RWX; 826 827 /* 828 * Initialize kernel pmap 829 */ 830 addr = round_page(vstart); 831 kpm = pmap_kernel(); 832 memset(kpm, 0, sizeof(*kpm)); 833 834 rw_init(&kpm->pm_obj_lock); 835 uvm_obj_init(&kpm->pm_obj, &pmap_pager, false, 1); 836 uvm_obj_setlock(&kpm->pm_obj, &kpm->pm_obj_lock); 837 838 kpm->pm_space = HPPA_SID_KERNEL; 839 kpm->pm_pid = HPPA_PID_KERNEL; 840 kpm->pm_pdir_pg = NULL; 841 kpm->pm_pdir = (uint32_t *)addr; 842 843 memset((void *)addr, 0, PAGE_SIZE); 844 fdcache(HPPA_SID_KERNEL, addr, PAGE_SIZE); 845 addr += PAGE_SIZE; 846 847 /* 848 * Allocate various tables and structures. 849 */ 850 mtctl(addr, CR_VTOP); 851 hppa_vtop = addr; 852 size = round_page((hppa_sid_max + 1) * 4); 853 memset((void *)addr, 0, size); 854 fdcache(HPPA_SID_KERNEL, addr, size); 855 856 addr += size; 857 pmap_sdir_set(HPPA_SID_KERNEL, kpm->pm_pdir); 858 859 /* 860 * cpuid() found out how big the HPT should be, so align addr to 861 * what will be its beginning. We don't waste the pages skipped 862 * for the alignment. 863 */ 864 #ifdef USE_HPT 865 if (pmap_hptsize) { 866 struct hpt_entry *hptp; 867 int i, error; 868 869 if (addr & (pmap_hptsize - 1)) 870 addr += pmap_hptsize; 871 addr &= ~(pmap_hptsize - 1); 872 873 memset((void *)addr, 0, pmap_hptsize); 874 hptp = (struct hpt_entry *)addr; 875 for (i = pmap_hptsize / sizeof(struct hpt_entry); i--; ) { 876 hptp[i].hpt_valid = 0; 877 hptp[i].hpt_space = 0xffff; 878 hptp[i].hpt_vpn = 0; 879 } 880 pmap_hpt = addr; 881 addr += pmap_hptsize; 882 883 UVMHIST_LOG(maphist, "hpt_table %#jx @ %#jx", 884 pmap_hptsize, addr, 0, 0); 885 886 if ((error = (cpu_hpt_init)(pmap_hpt, pmap_hptsize)) < 0) { 887 printf("WARNING: HPT init error %d -- DISABLED\n", 888 error); 889 pmap_hpt = 0; 890 } else { 891 UVMHIST_LOG(maphist, 892 "HPT installed for %jd entries @ %#jx", 893 pmap_hptsize / sizeof(struct hpt_entry), addr, 0, 894 0); 895 } 896 } 897 #endif 898 899 /* Setup vtop in lwp0 trapframe. */ 900 lwp0.l_md.md_regs->tf_vtop = hppa_vtop; 901 902 /* Pre-allocate PDEs for kernel virtual */ 903 nkpdes = (VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / PDE_SIZE; 904 /* ... and io space too */ 905 nkpdes += HPPA_IOLEN / PDE_SIZE; 906 /* ... and all physmem (VA == PA) */ 907 npdes = nkpdes + (physmem + atop(PDE_SIZE) - 1) / atop(PDE_SIZE); 908 909 UVMHIST_LOG(maphist, "npdes %jd", npdes, 0, 0, 0); 910 911 /* map the pdes */ 912 for (va = 0; npdes--; va += PDE_SIZE, addr += PAGE_SIZE) { 913 /* last nkpdes are for the kernel virtual */ 914 if (npdes == nkpdes - 1) 915 va = SYSCALLGATE; 916 if (npdes == HPPA_IOLEN / PDE_SIZE - 1) 917 va = HPPA_IOBEGIN; 918 /* now map the pde for the physmem */ 919 memset((void *)addr, 0, PAGE_SIZE); 920 921 UVMHIST_LOG(maphist, "pde premap 0x%08jx 0x%08jx", va, 922 addr, 0, 0); 923 pmap_pde_set(kpm, va, addr); 924 kpm->pm_stats.resident_count++; /* count PTP as resident */ 925 } 926 927 /* 928 * At this point we've finished reserving memory for the kernel. 929 */ 930 /* XXXNH */ 931 resvphysmem = atop(addr); 932 933 ksrx = (paddr_t) &kernel_text; 934 kerx = (paddr_t) &etext; 935 ksro = (paddr_t) &__rodata_start; 936 kero = (paddr_t) &__rodata_end; 937 ksrw = (paddr_t) &__data_start; 938 kerw = addr; 939 940 /* 941 * The kernel text, data, and bss must be direct-mapped, 942 * because the kernel often runs in physical mode, and 943 * anyways the loader loaded the kernel into physical 944 * memory exactly where it was linked. 945 * 946 * All memory already allocated after bss, either by 947 * our caller or by this function itself, must also be 948 * direct-mapped, because it's completely unmanaged 949 * and was allocated in physical mode. 950 * 951 * BTLB entries are used to do this direct mapping. 952 * BTLB entries have a minimum and maximum possible size, 953 * and MD code gives us these sizes in units of pages. 954 */ 955 956 btlb_entry_min = (vsize_t) hppa_btlb_size_min * PAGE_SIZE; 957 btlb_entry_max = (vsize_t) hppa_btlb_size_max * PAGE_SIZE; 958 959 /* 960 * To try to conserve BTLB entries, take a hint from how 961 * the kernel was linked: take the kernel text start as 962 * our effective minimum BTLB entry size, assuming that 963 * the data segment was also aligned to that size. 964 * 965 * In practice, linking the kernel at 2MB, and aligning 966 * the data segment to a 2MB boundary, should control well 967 * how much of the BTLB the pmap uses. However, this code 968 * should not rely on this 2MB magic number, nor should 969 * it rely on the data segment being aligned at all. This 970 * is to allow (smaller) kernels (linked lower) to work fine. 971 */ 972 btlb_entry_min = (vaddr_t) &kernel_text; 973 974 if (usebtlb) { 975 #define BTLB_SET_SIZE 16 976 vaddr_t btlb_entry_start[BTLB_SET_SIZE]; 977 vsize_t btlb_entry_size[BTLB_SET_SIZE]; 978 int btlb_entry_vm_prot[BTLB_SET_SIZE]; 979 int btlb_i; 980 int btlb_j; 981 982 /* 983 * Now make BTLB entries to direct-map the kernel text 984 * read- and execute-only as much as possible. Note that 985 * if the data segment isn't nicely aligned, the last 986 * BTLB entry for the kernel text may also cover some of 987 * the data segment, meaning it will have to allow writing. 988 */ 989 addr = ksrx; 990 991 UVMHIST_LOG(maphist, 992 "BTLB mapping text and rodata @ %#jx - %#jx", addr, kero, 993 0, 0); 994 995 btlb_j = 0; 996 while (addr < (vaddr_t) kero) { 997 998 /* Set up the next BTLB entry. */ 999 KASSERT(btlb_j < BTLB_SET_SIZE); 1000 btlb_entry_start[btlb_j] = addr; 1001 btlb_entry_size[btlb_j] = btlb_entry_min; 1002 btlb_entry_vm_prot[btlb_j] = 1003 VM_PROT_READ | VM_PROT_EXECUTE; 1004 if (addr + btlb_entry_min > kero) 1005 btlb_entry_vm_prot[btlb_j] |= VM_PROT_WRITE; 1006 1007 /* Coalesce BTLB entries whenever possible. */ 1008 while (btlb_j > 0 && 1009 btlb_entry_vm_prot[btlb_j] == 1010 btlb_entry_vm_prot[btlb_j - 1] && 1011 btlb_entry_size[btlb_j] == 1012 btlb_entry_size[btlb_j - 1] && 1013 !(btlb_entry_start[btlb_j - 1] & 1014 ((btlb_entry_size[btlb_j - 1] << 1) - 1)) && 1015 (btlb_entry_size[btlb_j - 1] << 1) <= 1016 btlb_entry_max) 1017 btlb_entry_size[--btlb_j] <<= 1; 1018 1019 /* Move on. */ 1020 addr = 1021 btlb_entry_start[btlb_j] + btlb_entry_size[btlb_j]; 1022 btlb_j++; 1023 } 1024 1025 /* 1026 * Now make BTLB entries to direct-map the kernel data, 1027 * bss, and all of the preallocated space read-write. 1028 * 1029 * Note that, unlike above, we're not concerned with 1030 * making these BTLB entries such that they finish as 1031 * close as possible to the end of the space we need 1032 * them to map. Instead, to minimize the number of BTLB 1033 * entries we need, we make them as large as possible. 1034 * The only thing this wastes is kernel virtual space, 1035 * which is plentiful. 1036 */ 1037 1038 UVMHIST_LOG(maphist, "mapping data, bss, etc @ %#jx - %#jx", 1039 addr, kerw, 0, 0); 1040 1041 while (addr < kerw) { 1042 1043 /* Make the next BTLB entry. */ 1044 KASSERT(btlb_j < BTLB_SET_SIZE); 1045 size = btlb_entry_min; 1046 while ((addr + size) < kerw && 1047 (size << 1) < btlb_entry_max && 1048 !(addr & ((size << 1) - 1))) 1049 size <<= 1; 1050 btlb_entry_start[btlb_j] = addr; 1051 btlb_entry_size[btlb_j] = size; 1052 btlb_entry_vm_prot[btlb_j] = 1053 VM_PROT_READ | VM_PROT_WRITE; 1054 1055 /* Move on. */ 1056 addr = 1057 btlb_entry_start[btlb_j] + btlb_entry_size[btlb_j]; 1058 btlb_j++; 1059 } 1060 1061 /* Now insert all of the BTLB entries. */ 1062 for (btlb_i = 0; btlb_i < btlb_j; btlb_i++) { 1063 int error; 1064 int prot; 1065 1066 btlb_entry_got = btlb_entry_size[btlb_i]; 1067 prot = btlb_entry_vm_prot[btlb_i]; 1068 1069 error = hppa_btlb_insert(kpm->pm_space, 1070 btlb_entry_start[btlb_i], btlb_entry_start[btlb_i], 1071 &btlb_entry_got, 1072 kpm->pm_pid | pmap_prot(kpm, prot)); 1073 1074 if (error) 1075 panic("%s: cannot insert BTLB entry", 1076 __func__); 1077 if (btlb_entry_got != btlb_entry_size[btlb_i]) 1078 panic("%s: BTLB entry mapped wrong amount", 1079 __func__); 1080 } 1081 1082 kerw = 1083 btlb_entry_start[btlb_j - 1] + btlb_entry_size[btlb_j - 1]; 1084 } 1085 1086 /* 1087 * We now know the exact beginning of managed kernel virtual space. 1088 * 1089 * Finally, load physical pages into UVM. There are three segments of 1090 * pages. 1091 */ 1092 1093 availphysmem = 0; 1094 1095 pmap_page_physload(resvmem, atop(ksrx)); 1096 pmap_page_physload(atop(kero), atop(ksrw)); 1097 pmap_page_physload(atop(kerw), physmem); 1098 1099 mutex_init(&pmaps_lock, MUTEX_DEFAULT, IPL_NONE); 1100 1101 /* TODO optimize/inline the kenter */ 1102 for (va = PAGE_SIZE; va < ptoa(physmem); va += PAGE_SIZE) { 1103 vm_prot_t prot = UVM_PROT_RW; 1104 1105 if (va < resvmem) 1106 prot = UVM_PROT_RX; 1107 else if (va >= ksrx && va < kerx) 1108 prot = UVM_PROT_RX; 1109 else if (va >= ksro && va < kero) 1110 prot = UVM_PROT_R; 1111 #ifdef DIAGNOSTIC 1112 else if (va == uvm_lwp_getuarea(&lwp0) + USPACE - PAGE_SIZE) 1113 prot = UVM_PROT_NONE; 1114 #endif 1115 pmap_kenter_pa(va, va, prot, PMAP_DIRECTMAP); 1116 } 1117 1118 /* XXXNH update */ 1119 UVMHIST_LOG(maphist, "mapped %#jx - %#jx", ksro, kero, 0, 0); 1120 UVMHIST_LOG(maphist, "mapped %#jx - %#jx", ksrw, kerw, 0, 0); 1121 1122 } 1123 1124 /* 1125 * Finishes the initialization of the pmap module. 1126 * This procedure is called from uvm_init() in uvm/uvm_init.c 1127 * to initialize any remaining data structures that the pmap module 1128 * needs to map virtual memory (VM is already ON). 1129 */ 1130 void 1131 pmap_init(void) 1132 { 1133 extern void gateway_page(void); 1134 volatile pt_entry_t *pde; 1135 int i; 1136 1137 UVMHIST_FUNC(__func__) 1138 UVMHIST_CALLED(maphist); 1139 1140 sid_counter = HPPA_SID_KERNEL; 1141 1142 pool_init(&pmap_pool, sizeof(struct pmap), 0, 0, 0, "pmappl", 1143 &pool_allocator_nointr, IPL_NONE); 1144 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pmappv", 1145 &pool_allocator_nointr, IPL_NONE); 1146 1147 pool_setlowat(&pmap_pv_pool, pmap_pvlowat); 1148 pool_sethiwat(&pmap_pv_pool, pmap_pvlowat * 32); 1149 1150 /* 1151 * map SysCall gateway page once for everybody 1152 * NB: we'll have to remap the phys memory 1153 * if we have any at SYSCALLGATE address (; 1154 * 1155 * no spls since no interrupts 1156 */ 1157 if (!(pde = pmap_pde_get(pmap_kernel()->pm_pdir, SYSCALLGATE)) && 1158 !(pde = pmap_pde_alloc(pmap_kernel(), SYSCALLGATE, NULL))) 1159 panic("pmap_init: cannot allocate pde"); 1160 1161 pmap_pte_set(pde, SYSCALLGATE, (paddr_t)&gateway_page | 1162 PTE_PROT(TLB_GATE_PROT)); 1163 1164 for (i = 0; i < __arraycount(pmap_pv_locks); i++) 1165 mutex_init(&pmap_pv_locks[i].lock, MUTEX_DEFAULT, IPL_VM); 1166 1167 pmap_initialized = true; 1168 1169 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 1170 } 1171 1172 /* 1173 * How much virtual space does this kernel have? 1174 */ 1175 void 1176 pmap_virtual_space(vaddr_t *startp, vaddr_t *endp) 1177 { 1178 1179 *startp = SYSCALLGATE + PAGE_SIZE; 1180 *endp = VM_MAX_KERNEL_ADDRESS; 1181 } 1182 1183 /* 1184 * pmap_create() 1185 * 1186 * Create and return a physical map. 1187 * The map is an actual physical map, and may be referenced by the hardware. 1188 */ 1189 pmap_t 1190 pmap_create(void) 1191 { 1192 pmap_t pmap; 1193 pa_space_t space; 1194 1195 UVMHIST_FUNC(__func__) 1196 UVMHIST_CALLED(maphist); 1197 1198 pmap = pool_get(&pmap_pool, PR_WAITOK); 1199 1200 UVMHIST_LOG(maphist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0); 1201 1202 rw_init(&pmap->pm_obj_lock); 1203 uvm_obj_init(&pmap->pm_obj, &pmap_pager, false, 1); 1204 uvm_obj_setlock(&pmap->pm_obj, &pmap->pm_obj_lock); 1205 1206 mutex_enter(&pmaps_lock); 1207 1208 /* 1209 * Allocate space IDs for the pmap; we get the protection ID from this. 1210 * If all are allocated, there is nothing we can do. 1211 */ 1212 /* XXXNH can't this loop forever??? */ 1213 for (space = sid_counter; pmap_sdir_get(space); 1214 space = (space + 1) % hppa_sid_max) 1215 ; 1216 1217 if ((pmap->pm_pdir_pg = pmap_pagealloc(NULL, 0)) == NULL) 1218 panic("pmap_create: no pages"); 1219 pmap->pm_ptphint = NULL; 1220 pmap->pm_pdir = (uint32_t *)VM_PAGE_TO_PHYS(pmap->pm_pdir_pg); 1221 pmap_sdir_set(space, pmap->pm_pdir); 1222 1223 pmap->pm_space = space; 1224 pmap->pm_pid = (space + 1) << 1; 1225 1226 pmap->pm_stats.resident_count = 1; 1227 pmap->pm_stats.wired_count = 0; 1228 1229 mutex_exit(&pmaps_lock); 1230 1231 UVMHIST_LOG(maphist, "pm %#jx, space %jd, pid %jd", 1232 (uintptr_t)pmap, space, pmap->pm_pid, 0); 1233 1234 return (pmap); 1235 } 1236 1237 /* 1238 * pmap_destroy(pmap) 1239 * Gives up a reference to the specified pmap. When the reference count 1240 * reaches zero the pmap structure is added to the pmap free list. 1241 * Should only be called if the map contains no valid mappings. 1242 */ 1243 void 1244 pmap_destroy(pmap_t pmap) 1245 { 1246 UVMHIST_FUNC(__func__) 1247 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0); 1248 1249 #ifdef DIAGNOSTIC 1250 struct uvm_page_array a; 1251 struct vm_page *pg; 1252 off_t off; 1253 #endif 1254 1255 membar_release(); 1256 if (atomic_dec_uint_nv(&pmap->pm_obj.uo_refs) > 0) 1257 return; 1258 membar_acquire(); 1259 1260 #ifdef DIAGNOSTIC 1261 uvm_page_array_init(&a, &pmap->pm_obj, 0); 1262 off = 0; 1263 rw_enter(pmap->pm_lock, RW_WRITER); 1264 while ((pg = uvm_page_array_fill_and_peek(&a, off, 0)) != NULL) { 1265 pt_entry_t *pde, *epde; 1266 struct vm_page *spg; 1267 struct pv_entry *pv, *npv; 1268 paddr_t pa; 1269 vaddr_t va; 1270 1271 off = pg->offset + PAGE_SIZE; 1272 uvm_page_array_advance(&a); 1273 KASSERT(pg != pmap->pm_pdir_pg); 1274 pa = VM_PAGE_TO_PHYS(pg); 1275 1276 UVMHIST_LOG(maphist, "pm %#jx: stray ptp %#jx w/ %jd entries:", 1277 (uintptr_t)pmap, pa, pg->wire_count - 1, 0); 1278 1279 pde = (pt_entry_t *)pa; 1280 epde = (pt_entry_t *)(pa + PAGE_SIZE); 1281 for (; pde < epde; pde++) { 1282 if (*pde == 0) 1283 continue; 1284 1285 spg = PHYS_TO_VM_PAGE(PTE_PAGE(*pde)); 1286 if (spg == NULL) 1287 continue; 1288 1289 struct vm_page_md * const md = VM_PAGE_TO_MD(spg); 1290 pmap_pv_lock(md); 1291 for (pv = md->pvh_list; pv != NULL; pv = npv) { 1292 npv = pv->pv_next; 1293 if (pv->pv_pmap != pmap) 1294 continue; 1295 1296 UVMHIST_LOG(maphist, " %#jx", pv->pv_va, 0, 0, 1297 0); 1298 1299 va = pv->pv_va & PV_VAMASK; 1300 pmap_pv_unlock(md); 1301 pmap_remove(pmap, va, va + PAGE_SIZE); 1302 pmap_pv_lock(md); 1303 /* List may have changed: restart. */ 1304 npv = md->pvh_list; 1305 } 1306 pmap_pv_unlock(md); 1307 } 1308 } 1309 rw_exit(pmap->pm_lock); 1310 uvm_page_array_fini(&a); 1311 #endif 1312 pmap_sdir_set(pmap->pm_space, 0); 1313 rw_enter(pmap->pm_lock, RW_WRITER); 1314 pmap_pagefree(pmap->pm_pdir_pg); 1315 rw_exit(pmap->pm_lock); 1316 1317 uvm_obj_destroy(&pmap->pm_obj, false); 1318 rw_destroy(&pmap->pm_obj_lock); 1319 pool_put(&pmap_pool, pmap); 1320 } 1321 1322 /* 1323 * Add a reference to the specified pmap. 1324 */ 1325 void 1326 pmap_reference(pmap_t pmap) 1327 { 1328 UVMHIST_FUNC(__func__) 1329 UVMHIST_CALLARGS(maphist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0); 1330 1331 atomic_inc_uint(&pmap->pm_obj.uo_refs); 1332 } 1333 1334 1335 void 1336 pmap_syncicache_page(struct vm_page *pg, pmap_t pm, vaddr_t va) 1337 { 1338 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1339 struct pv_entry *pve = md->pvh_list; 1340 1341 for (; pve; pve = pve->pv_next) { 1342 pmap_t fpm = pve->pv_pmap; 1343 vaddr_t fva = pve->pv_va & PV_VAMASK; 1344 pt_entry_t pte = pmap_vp_find(fpm, fva); 1345 1346 if ((pte & PTE_PROT(TLB_DIRTY)) == 0) 1347 continue; 1348 1349 /* Don't attempt to use the mapping we're adding */ 1350 if (pm == fpm && va == fva) 1351 continue; 1352 1353 fdcache(fpm->pm_space, fva, PAGE_SIZE); 1354 ficache(fpm->pm_space, fva, PAGE_SIZE); 1355 break; 1356 } 1357 } 1358 1359 /* 1360 * pmap_enter(pmap, va, pa, prot, flags) 1361 * Create a translation for the virtual address (va) to the physical 1362 * address (pa) in the pmap with the protection requested. If the 1363 * translation is wired then we can not allow a page fault to occur 1364 * for this mapping. 1365 */ 1366 int 1367 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1368 { 1369 volatile pt_entry_t *pde; 1370 pt_entry_t pte; 1371 struct vm_page *pg = NULL, *ptp = NULL; 1372 struct pv_entry *pve = NULL; 1373 bool wired = (flags & PMAP_WIRED) != 0; 1374 1375 UVMHIST_FUNC(__func__); 1376 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx pa %#jx prot %#jx", 1377 (uintptr_t)pmap, va, pa, prot); 1378 UVMHIST_LOG(maphist, "...flags %#jx", flags, 0, 0, 0); 1379 1380 pmap_lock(pmap); 1381 1382 if (!(pde = pmap_pde_get(pmap->pm_pdir, va)) && 1383 !(pde = pmap_pde_alloc(pmap, va, &ptp))) { 1384 if (flags & PMAP_CANFAIL) { 1385 pmap_unlock(pmap); 1386 return (ENOMEM); 1387 } 1388 1389 panic("pmap_enter: cannot allocate pde"); 1390 } 1391 1392 if (!ptp) 1393 ptp = pmap_pde_ptp(pmap, pde); 1394 1395 if ((pte = pmap_pte_get(pde, va))) { 1396 UVMHIST_LOG(maphist, "remapping %#jx -> %#jx", pte, pa, 0, 0); 1397 1398 pmap_pte_flush(pmap, va, pte); 1399 if (wired && !(pte & PTE_PROT(TLB_WIRED))) 1400 pmap->pm_stats.wired_count++; 1401 else if (!wired && (pte & PTE_PROT(TLB_WIRED))) 1402 pmap->pm_stats.wired_count--; 1403 1404 pg = PHYS_TO_VM_PAGE(PTE_PAGE(pte)); 1405 if (PTE_PAGE(pte) == pa) { 1406 UVMHIST_LOG(maphist, "same page", 0, 0, 0, 0); 1407 goto enter; 1408 } 1409 1410 if (pg != NULL) { 1411 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1412 1413 pmap_pv_lock(md); 1414 pve = pmap_pv_remove(pg, pmap, va); 1415 md->pvh_attrs |= pmap_pvh_attrs(pte); 1416 pmap_pv_unlock(md); 1417 } 1418 } else { 1419 UVMHIST_LOG(maphist, "new mapping %#jx -> %#jx", 1420 va, pa, 0, 0); 1421 pte = PTE_PROT(TLB_REFTRAP); 1422 pmap->pm_stats.resident_count++; 1423 if (wired) 1424 pmap->pm_stats.wired_count++; 1425 if (ptp) 1426 ptp->wire_count++; 1427 } 1428 1429 if (pmap_initialized && (pg = PHYS_TO_VM_PAGE(pa))) { 1430 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1431 1432 if (!pve && !(pve = pmap_pv_alloc())) { 1433 if (flags & PMAP_CANFAIL) { 1434 pmap_unlock(pmap); 1435 return (ENOMEM); 1436 } 1437 panic("%s: no pv entries available", __func__); 1438 } 1439 pte |= PTE_PROT(pmap_prot(pmap, prot)); 1440 pmap_resolve_alias(pg, pmap, va, pte); 1441 1442 pmap_pv_lock(md); 1443 pmap_pv_enter(pg, pve, pmap, va, ptp, 0); 1444 pmap_pv_unlock(md); 1445 } else if (pve) { 1446 pmap_pv_free(pve); 1447 } 1448 1449 enter: 1450 /* preserve old ref & mod */ 1451 pte = pa | PTE_PROT(pmap_prot(pmap, prot)) | 1452 (pte & PTE_PROT(TLB_UNCACHEABLE|TLB_DIRTY|TLB_REFTRAP)); 1453 1454 if (pg != NULL) { 1455 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1456 1457 if ((pte & PTE_PROT(TLB_EXECUTE)) != 0 && 1458 !IS_PVFEXEC_P(md->pvh_attrs)) { 1459 pmap_syncicache_page(pg, pmap, va); 1460 md->pvh_attrs |= PVF_EXEC; 1461 } 1462 } 1463 1464 if (IS_IOPAGE_P(pa)) 1465 pte |= PTE_PROT(TLB_UNCACHEABLE); 1466 if (wired) 1467 pte |= PTE_PROT(TLB_WIRED); 1468 pmap_pte_set(pde, va, pte); 1469 1470 pmap_unlock(pmap); 1471 1472 UVMHIST_LOG(maphist, "<--- done (0)", 0, 0, 0, 0); 1473 1474 return (0); 1475 } 1476 1477 /* 1478 * pmap_remove(pmap, sva, eva) 1479 * unmaps all virtual addresses in the virtual address 1480 * range determined by [sva, eva) and pmap. 1481 * sva and eva must be on machine independent page boundaries and 1482 * sva must be less than or equal to eva. 1483 */ 1484 void 1485 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1486 { 1487 UVMHIST_FUNC(__func__); 1488 UVMHIST_CALLARGS(maphist, "sva %#jx eva %#jx", sva, eva, 0, 0); 1489 1490 struct pv_entry *pve; 1491 volatile pt_entry_t *pde = NULL; 1492 pt_entry_t pte; 1493 struct vm_page *pg, *ptp; 1494 vaddr_t pdemask; 1495 int batch; 1496 1497 pmap_lock(pmap); 1498 1499 for (batch = 0; sva < eva; sva += PAGE_SIZE) { 1500 pdemask = sva & PDE_MASK; 1501 if (!(pde = pmap_pde_get(pmap->pm_pdir, sva))) { 1502 sva = pdemask + PDE_SIZE - PAGE_SIZE; 1503 continue; 1504 } 1505 batch = pdemask == sva && sva + PDE_SIZE <= eva; 1506 1507 if ((pte = pmap_pte_get(pde, sva))) { 1508 1509 /* TODO measure here the speed tradeoff 1510 * for flushing whole 4M vs per-page 1511 * in case of non-complete pde fill 1512 */ 1513 pmap_pte_flush(pmap, sva, pte); 1514 if (pte & PTE_PROT(TLB_WIRED)) 1515 pmap->pm_stats.wired_count--; 1516 pmap->pm_stats.resident_count--; 1517 1518 /* iff properly accounted pde will be dropped anyway */ 1519 if (!batch) 1520 pmap_pte_set(pde, sva, 0); 1521 1522 if (pmap_initialized && 1523 (pg = PHYS_TO_VM_PAGE(PTE_PAGE(pte)))) { 1524 struct vm_page_md * const md = 1525 VM_PAGE_TO_MD(pg); 1526 1527 pmap_pv_lock(md); 1528 pve = pmap_pv_remove(pg, pmap, sva); 1529 md->pvh_attrs |= pmap_pvh_attrs(pte); 1530 pmap_pv_unlock(md); 1531 1532 if (pve != NULL) 1533 pmap_pv_free(pve); 1534 } else { 1535 if (IS_IOPAGE_P(PTE_PAGE(pte))) { 1536 ptp = pmap_pde_ptp(pmap, pde); 1537 if (ptp != NULL) 1538 pmap_pde_release(pmap, sva, 1539 ptp); 1540 } 1541 } 1542 } 1543 } 1544 1545 pmap_unlock(pmap); 1546 1547 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 1548 } 1549 1550 void 1551 pmap_write_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1552 { 1553 UVMHIST_FUNC(__func__); 1554 UVMHIST_CALLARGS(maphist, "pm %#jx sva %#jx eva %#jx prot %#jx", 1555 (uintptr_t)pmap, sva, eva, prot); 1556 1557 struct vm_page *pg; 1558 volatile pt_entry_t *pde = NULL; 1559 pt_entry_t pte; 1560 u_int pteprot, pdemask; 1561 1562 sva = trunc_page(sva); 1563 pteprot = PTE_PROT(pmap_prot(pmap, prot)); 1564 1565 pmap_lock(pmap); 1566 1567 for (pdemask = 1; sva < eva; sva += PAGE_SIZE) { 1568 if (pdemask != (sva & PDE_MASK)) { 1569 pdemask = sva & PDE_MASK; 1570 if (!(pde = pmap_pde_get(pmap->pm_pdir, sva))) { 1571 sva = pdemask + PDE_SIZE - PAGE_SIZE; 1572 continue; 1573 } 1574 } 1575 if ((pte = pmap_pte_get(pde, sva))) { 1576 UVMHIST_LOG(maphist, "va% #jx pte %#jx", sva, pte, 1577 0, 0); 1578 /* 1579 * Determine if mapping is changing. 1580 * If not, nothing to do. 1581 */ 1582 if ((pte & PTE_PROT(TLB_AR_MASK)) == pteprot) 1583 continue; 1584 1585 pg = PHYS_TO_VM_PAGE(PTE_PAGE(pte)); 1586 if (pg != NULL) { 1587 struct vm_page_md * const md = 1588 VM_PAGE_TO_MD(pg); 1589 1590 pmap_pv_lock(md); 1591 md->pvh_attrs |= pmap_pvh_attrs(pte); 1592 pmap_pv_unlock(md); 1593 } 1594 1595 /* Add TLB_EXECUTE if PVF_EXEC ??? */ 1596 pmap_pte_flush(pmap, sva, pte); 1597 pte &= ~PTE_PROT(TLB_AR_MASK); 1598 pte |= pteprot; 1599 pmap_pte_set(pde, sva, pte); 1600 } 1601 } 1602 1603 pmap_unlock(pmap); 1604 } 1605 1606 void 1607 pmap_page_remove(struct vm_page *pg) 1608 { 1609 UVMHIST_FUNC(__func__) 1610 UVMHIST_CALLARGS(maphist, "pg %#jx", (uintptr_t)pg, 0, 0, 0); 1611 1612 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1613 struct pv_entry *pve, *npve, **pvp; 1614 1615 if (md->pvh_list == NULL) { 1616 KASSERT((md->pvh_attrs & PVF_EXEC) == 0); 1617 return; 1618 } 1619 1620 restart: 1621 pmap_pv_lock(md); 1622 pvp = &md->pvh_list; 1623 for (pve = md->pvh_list; pve; pve = npve) { 1624 pmap_t pmap = pve->pv_pmap; 1625 vaddr_t va = pve->pv_va & PV_VAMASK; 1626 volatile pt_entry_t *pde; 1627 pt_entry_t pte; 1628 bool locked; 1629 1630 UVMHIST_LOG(maphist, "... pm %#jx va %#jx", (uintptr_t)pmap, 1631 va, 0, 0); 1632 1633 npve = pve->pv_next; 1634 if (pve->pv_va & PV_KENTER) { 1635 /* 1636 * This is an unmanaged mapping, it must be preserved. 1637 * Move it back on the list and advance the end-of-list 1638 * pointer. 1639 */ 1640 *pvp = pve; 1641 pvp = &pve->pv_next; 1642 1643 /* Get the pte for this mapping */ 1644 pde = pmap_pde_get(pmap->pm_pdir, va); 1645 pte = pmap_pte_get(pde, va); 1646 } else { 1647 /* 1648 * We have to seamlessly get a hold on the pmap's lock 1649 * while holding the PV head lock, to know that the 1650 * mapping is still in place and we can operate on it. 1651 * If that can't be had, drop the PV head lock, wait 1652 * for the pmap's lock to become available, and then 1653 * try again. 1654 */ 1655 UVMHIST_LOG(maphist, "... pm %#jx va %#jx... removing", 1656 (uintptr_t)pmap, va, 0, 0); 1657 1658 pmap_reference(pmap); 1659 locked = pmap_trylock(pmap); 1660 if (!locked) { 1661 pmap_pv_unlock(md); 1662 pmap_lock(pmap); 1663 /* nothing */ 1664 pmap_unlock(pmap); 1665 pmap_destroy(pmap); 1666 1667 UVMHIST_LOG(maphist, "... failed lock", 0, 0, 0, 1668 0); 1669 goto restart; 1670 } 1671 pde = pmap_pde_get(pmap->pm_pdir, va); 1672 pte = pmap_pte_get(pde, va); 1673 1674 md->pvh_attrs |= pmap_pvh_attrs(pte); 1675 } 1676 1677 pmap_pte_flush(pmap, va, pte); 1678 if (pte & PTE_PROT(TLB_WIRED)) 1679 pmap->pm_stats.wired_count--; 1680 pmap->pm_stats.resident_count--; 1681 1682 if (!(pve->pv_va & PV_KENTER)) { 1683 pmap_pte_set(pde, va, 0); 1684 1685 pmap_pv_unlock(md); 1686 pmap_pv_free(pve); 1687 pmap_unlock(pmap); 1688 pmap_destroy(pmap); 1689 UVMHIST_LOG(maphist, "... removed", 0, 0, 0, 0); 1690 *pvp = npve; 1691 goto restart; 1692 } 1693 } 1694 md->pvh_attrs &= ~PVF_EXEC; 1695 *pvp = NULL; 1696 1697 pmap_pv_unlock(md); 1698 1699 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 1700 } 1701 1702 /* 1703 * Routine: pmap_unwire 1704 * Function: Change the wiring attribute for a map/virtual-address 1705 * pair. 1706 * In/out conditions: 1707 * The mapping must already exist in the pmap. 1708 * 1709 * Change the wiring for a given virtual page. This routine currently is 1710 * only used to unwire pages and hence the mapping entry will exist. 1711 */ 1712 void 1713 pmap_unwire(pmap_t pmap, vaddr_t va) 1714 { 1715 UVMHIST_FUNC(__func__); 1716 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx", (uintptr_t)pmap, va, 0, 0); 1717 1718 volatile pt_entry_t *pde; 1719 pt_entry_t pte = 0; 1720 1721 pmap_lock(pmap); 1722 if ((pde = pmap_pde_get(pmap->pm_pdir, va))) { 1723 pte = pmap_pte_get(pde, va); 1724 1725 KASSERT(pte); 1726 1727 if (pte & PTE_PROT(TLB_WIRED)) { 1728 pte &= ~PTE_PROT(TLB_WIRED); 1729 pmap->pm_stats.wired_count--; 1730 pmap_pte_set(pde, va, pte); 1731 } 1732 } 1733 pmap_unlock(pmap); 1734 1735 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 1736 } 1737 1738 bool 1739 pmap_changebit(struct vm_page *pg, u_int set, u_int clear) 1740 { 1741 UVMHIST_FUNC(__func__); 1742 UVMHIST_CALLARGS(maphist, "pg %#jx (md %#jx) set %#jx clear %#jx", 1743 (uintptr_t)pg, (uintptr_t)VM_PAGE_TO_MD(pg), set, clear); 1744 1745 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1746 struct pv_entry *pve; 1747 int res; 1748 1749 KASSERT((set & clear) == 0); 1750 KASSERT((set & ~(PVF_REF|PVF_UNCACHEABLE)) == 0); 1751 KASSERT((clear & ~(PVF_MOD|PVF_WRITE|PVF_UNCACHEABLE)) == 0); 1752 1753 /* preserve other bits */ 1754 pmap_pv_lock(md); 1755 res = md->pvh_attrs & (set | clear); 1756 md->pvh_attrs ^= res; 1757 1758 for (pve = md->pvh_list; pve; pve = pve->pv_next) { 1759 pmap_t pmap = pve->pv_pmap; 1760 vaddr_t va = pve->pv_va & PV_VAMASK; 1761 volatile pt_entry_t *pde; 1762 pt_entry_t opte, pte; 1763 1764 if ((pde = pmap_pde_get(pmap->pm_pdir, va))) { 1765 opte = pte = pmap_pte_get(pde, va); 1766 #ifdef DEBUG 1767 if (!pte) { 1768 UVMHIST_LOG(maphist, "zero pte for %#jx", 1769 va, 0, 0, 0); 1770 continue; 1771 } 1772 #endif 1773 pte &= ~clear; 1774 pte |= set; 1775 1776 if (!(pve->pv_va & PV_KENTER)) { 1777 md->pvh_attrs |= pmap_pvh_attrs(pte); 1778 res |= pmap_pvh_attrs(opte); 1779 } 1780 1781 if (opte != pte) { 1782 pmap_pte_flush(pmap, va, opte); 1783 pmap_pte_set(pde, va, pte); 1784 } 1785 } 1786 } 1787 pmap_pv_unlock(md); 1788 1789 return ((res & (clear | set)) != 0); 1790 } 1791 1792 bool 1793 pmap_testbit(struct vm_page *pg, u_int bit) 1794 { 1795 UVMHIST_FUNC(__func__); 1796 UVMHIST_CALLARGS(maphist, "pg %#jx (md %#jx) bit %#jx", 1797 (uintptr_t)pg, (uintptr_t)VM_PAGE_TO_MD(pg), bit, 0); 1798 1799 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1800 struct pv_entry *pve; 1801 pt_entry_t pte; 1802 int ret; 1803 1804 pmap_pv_lock(md); 1805 for (pve = md->pvh_list; !(md->pvh_attrs & bit) && pve; 1806 pve = pve->pv_next) { 1807 pmap_t pm = pve->pv_pmap; 1808 1809 pte = pmap_vp_find(pm, pve->pv_va & PV_VAMASK); 1810 if (pve->pv_va & PV_KENTER) 1811 continue; 1812 1813 md->pvh_attrs |= pmap_pvh_attrs(pte); 1814 } 1815 ret = ((md->pvh_attrs & bit) != 0); 1816 pmap_pv_unlock(md); 1817 1818 return ret; 1819 } 1820 1821 /* 1822 * pmap_extract(pmap, va, pap) 1823 * fills in the physical address corresponding to the 1824 * virtual address specified by pmap and va into the 1825 * storage pointed to by pap and returns true if the 1826 * virtual address is mapped. returns false in not mapped. 1827 */ 1828 bool 1829 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1830 { 1831 UVMHIST_FUNC(__func__); 1832 UVMHIST_CALLARGS(maphist, "pm %#jx va %#jx", (uintptr_t)pmap, va, 0, 0); 1833 1834 pt_entry_t pte; 1835 1836 1837 if (pmap != pmap_kernel()) { 1838 pmap_lock(pmap); 1839 pte = pmap_vp_find(pmap, va); 1840 pmap_unlock(pmap); 1841 } else { 1842 pte = pmap_vp_find(pmap, va); 1843 } 1844 1845 if (pte) { 1846 if (pap) 1847 *pap = (pte & ~PGOFSET) | (va & PGOFSET); 1848 return true; 1849 } 1850 1851 return false; 1852 } 1853 1854 /* 1855 * pmap_activate(lwp) 1856 * 1857 * Activates the vmspace for the given LWP. 1858 * This is not necessarily the current LWP. 1859 */ 1860 void 1861 pmap_activate(struct lwp *l) 1862 { 1863 struct proc *p = l->l_proc; 1864 pmap_t pmap = p->p_vmspace->vm_map.pmap; 1865 pa_space_t space = pmap->pm_space; 1866 struct pcb *pcb = lwp_getpcb(l); 1867 1868 /* space is cached for the copy{in,out}'s pleasure */ 1869 pcb->pcb_space = space; 1870 fdcache(HPPA_SID_KERNEL, (vaddr_t)pcb, sizeof(struct pcb)); 1871 1872 if (p == curproc) 1873 mtctl(pmap->pm_pid, CR_PIDR2); 1874 } 1875 1876 void 1877 pmap_procwr(struct proc *p, vaddr_t va, size_t len) 1878 { 1879 const pmap_t pmap = p->p_vmspace->vm_map.pmap; 1880 const pa_space_t space = pmap->pm_space; 1881 1882 fdcache(space, va, len); 1883 ficache(space, va, len); 1884 pdtlb(space, va); 1885 pitlb(space, va); 1886 } 1887 1888 static inline void 1889 pmap_flush_page(struct vm_page *pg, bool purge) 1890 { 1891 UVMHIST_FUNC(__func__); 1892 UVMHIST_CALLARGS(maphist, "pg %#jx (md %#jx) purge %jd", 1893 (uintptr_t)pg, (uintptr_t)VM_PAGE_TO_MD(pg), purge, 0); 1894 1895 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 1896 struct pv_entry *pve; 1897 1898 /* purge cache for all possible mappings for the pa */ 1899 for (pve = md->pvh_list; pve; pve = pve->pv_next) { 1900 vaddr_t va = pve->pv_va & PV_VAMASK; 1901 pa_space_t sp = pve->pv_pmap->pm_space; 1902 1903 if (purge) 1904 pdcache(sp, va, PAGE_SIZE); 1905 else 1906 fdcache(sp, va, PAGE_SIZE); 1907 #if defined(HP8000_CPU) || defined(HP8200_CPU) || \ 1908 defined(HP8500_CPU) || defined(HP8600_CPU) 1909 ficache(sp, va, PAGE_SIZE); 1910 pdtlb(sp, va); 1911 pitlb(sp, va); 1912 #endif 1913 } 1914 } 1915 1916 /* 1917 * pmap_zero_page(pa) 1918 * 1919 * Zeros the specified page. 1920 */ 1921 void 1922 pmap_zero_page(paddr_t pa) 1923 { 1924 1925 UVMHIST_FUNC(__func__); 1926 UVMHIST_CALLARGS(maphist, "pa %#jx (pg %#jx)", pa, 1927 (uintptr_t)PHYS_TO_VM_PAGE(pa), 0, 0); 1928 1929 KASSERT(VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(pa))->pvh_list == NULL); 1930 KASSERT((VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(pa))->pvh_attrs & PVF_EXEC) == 0); 1931 1932 memset((void *)pa, 0, PAGE_SIZE); 1933 fdcache(HPPA_SID_KERNEL, pa, PAGE_SIZE); 1934 1935 #if defined(HP8000_CPU) || defined(HP8200_CPU) || \ 1936 defined(HP8500_CPU) || defined(HP8600_CPU) 1937 ficache(HPPA_SID_KERNEL, pa, PAGE_SIZE); 1938 pdtlb(HPPA_SID_KERNEL, pa); 1939 pitlb(HPPA_SID_KERNEL, pa); 1940 #endif 1941 } 1942 1943 /* 1944 * pmap_copy_page(src, dst) 1945 * 1946 * pmap_copy_page copies the source page to the destination page. 1947 */ 1948 void 1949 pmap_copy_page(paddr_t spa, paddr_t dpa) 1950 { 1951 UVMHIST_FUNC(__func__); 1952 UVMHIST_CALLARGS(maphist, "spa %#jx (pg %#jx) dpa %#jx (pg %#jx)", 1953 spa, (uintptr_t)PHYS_TO_VM_PAGE(spa), 1954 dpa, (uintptr_t)PHYS_TO_VM_PAGE(dpa)); 1955 1956 struct vm_page *srcpg = PHYS_TO_VM_PAGE(spa); 1957 1958 KASSERT(VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dpa))->pvh_list == NULL); 1959 KASSERT((VM_PAGE_TO_MD(PHYS_TO_VM_PAGE(dpa))->pvh_attrs & PVF_EXEC) == 0); 1960 1961 pmap_flush_page(srcpg, false); 1962 1963 memcpy((void *)dpa, (void *)spa, PAGE_SIZE); 1964 1965 pdcache(HPPA_SID_KERNEL, spa, PAGE_SIZE); 1966 fdcache(HPPA_SID_KERNEL, dpa, PAGE_SIZE); 1967 #if defined(HP8000_CPU) || defined(HP8200_CPU) || \ 1968 defined(HP8500_CPU) || defined(HP8600_CPU) 1969 ficache(HPPA_SID_KERNEL, spa, PAGE_SIZE); 1970 ficache(HPPA_SID_KERNEL, dpa, PAGE_SIZE); 1971 pdtlb(HPPA_SID_KERNEL, spa); 1972 pdtlb(HPPA_SID_KERNEL, dpa); 1973 pitlb(HPPA_SID_KERNEL, spa); 1974 pitlb(HPPA_SID_KERNEL, dpa); 1975 #endif 1976 } 1977 1978 void 1979 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1980 { 1981 UVMHIST_FUNC(__func__); 1982 if (va != 0) { 1983 UVMHIST_CALLARGS(maphist, "va %#jx pa %#jx prot %#jx flags %#jx", 1984 va, pa, prot, flags); 1985 } 1986 1987 volatile pt_entry_t *pde; 1988 pt_entry_t pte, opte; 1989 struct vm_page *pg; 1990 1991 if (!(pde = pmap_pde_get(pmap_kernel()->pm_pdir, va)) && 1992 !(pde = pmap_pde_alloc(pmap_kernel(), va, NULL))) 1993 panic("pmap_kenter_pa: cannot allocate pde for va=0x%lx", va); 1994 opte = pmap_pte_get(pde, va); 1995 pte = pa | PTE_PROT(TLB_WIRED | TLB_REFTRAP | 1996 pmap_prot(pmap_kernel(), prot & VM_PROT_ALL)); 1997 if (IS_IOPAGE_P(pa) || (flags & PMAP_NOCACHE)) 1998 pte |= PTE_PROT(TLB_UNCACHEABLE); 1999 2000 if ((flags & PMAP_DIRECTMAP) == 0) { 2001 pmap_kernel()->pm_stats.wired_count++; 2002 pmap_kernel()->pm_stats.resident_count++; 2003 } 2004 if (opte) 2005 pmap_pte_flush(pmap_kernel(), va, opte); 2006 2007 pg = pmap_initialized ? PHYS_TO_VM_PAGE(PTE_PAGE(pte)) : NULL; 2008 if (pg != NULL) { 2009 KASSERT(pa < HPPA_IOBEGIN); 2010 2011 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2012 struct pv_entry *pve; 2013 2014 pve = pmap_pv_alloc(); 2015 if (!pve) 2016 panic("%s: no pv entries available", __func__); 2017 UVMHIST_LOG(maphist, "va %#jx pa %#jx pte %#jx TLB_KENTER", 2018 va, pa, pte, 0); 2019 2020 pmap_resolve_alias(pg, pmap_kernel(), va, pte); 2021 2022 pmap_pv_lock(md); 2023 pmap_pv_enter(pg, pve, pmap_kernel(), va, NULL, PV_KENTER); 2024 pmap_pv_unlock(md); 2025 } 2026 pmap_pte_set(pde, va, pte); 2027 2028 if (va != 0) { 2029 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 2030 } 2031 2032 } 2033 2034 void 2035 pmap_kremove(vaddr_t va, vsize_t size) 2036 { 2037 UVMHIST_FUNC(__func__); 2038 bool pzero = false; 2039 if (va != 0) { 2040 UVMHIST_CALLARGS(maphist, "va %#jx...%#jx", va, va + size, 0, 2041 0); 2042 pzero = true; 2043 } 2044 2045 struct pv_entry *pve; 2046 vaddr_t eva, pdemask; 2047 volatile pt_entry_t *pde = NULL; 2048 pt_entry_t pte; 2049 struct vm_page *pg; 2050 pmap_t pmap = pmap_kernel(); 2051 2052 #ifdef DEBUG 2053 /* 2054 * Don't allow the VA == PA mappings, apart from page zero, to be 2055 * removed. Page zero is given special treatment so that we get TLB 2056 * faults when the kernel tries to de-reference NULL or anything else 2057 * in the first page when it shouldn't. 2058 */ 2059 if (va != 0 && va < ptoa(physmem)) { 2060 UVMHIST_LOG(maphist, "va %#jx size %#jx: unmapping physmem", va, 2061 size, 0, 0); 2062 return; 2063 } 2064 #endif 2065 2066 for (pdemask = 1, eva = va + size; va < eva; va += PAGE_SIZE) { 2067 if (pdemask != (va & PDE_MASK)) { 2068 pdemask = va & PDE_MASK; 2069 if (!(pde = pmap_pde_get(pmap->pm_pdir, va))) { 2070 va = pdemask + PDE_SIZE - PAGE_SIZE; 2071 continue; 2072 } 2073 } 2074 if (!(pte = pmap_pte_get(pde, va))) { 2075 UVMHIST_LOG(maphist, "unmapping unmapped %#jx", 2076 va, 0, 0, 0); 2077 continue; 2078 } 2079 2080 pmap_pte_flush(pmap, va, pte); 2081 pmap_pte_set(pde, va, 0); 2082 2083 pmap->pm_stats.wired_count--; 2084 pmap->pm_stats.resident_count--; 2085 2086 pg = pmap_initialized ? PHYS_TO_VM_PAGE(PTE_PAGE(pte)) : NULL; 2087 if (pg != NULL) { 2088 struct vm_page_md * const md = VM_PAGE_TO_MD(pg); 2089 2090 pmap_pv_lock(md); 2091 pve = pmap_pv_remove(pg, pmap, va); 2092 pmap_pv_unlock(md); 2093 2094 if (pve != NULL) 2095 pmap_pv_free(pve); 2096 } 2097 } 2098 if (pzero) { 2099 UVMHIST_LOG(maphist, "<--- done", 0, 0, 0, 0); 2100 } 2101 } 2102 2103 #if defined(USE_HPT) 2104 #if defined(DDB) 2105 /* 2106 * prints whole va->pa (aka HPT or HVT) 2107 */ 2108 void 2109 pmap_hptdump(void) 2110 { 2111 struct hpt_entry *hpt, *ehpt; 2112 2113 hpt = (struct hpt_entry *)pmap_hpt; 2114 ehpt = (struct hpt_entry *)((int)hpt + pmap_hptsize); 2115 db_printf("HPT dump %p-%p:\n", hpt, ehpt); 2116 for (; hpt < ehpt; hpt++) 2117 if (hpt->hpt_valid) { 2118 char buf[128]; 2119 2120 snprintb(buf, sizeof(buf), TLB_BITS, hpt->hpt_tlbprot); 2121 2122 db_printf("hpt@%p: %x{%sv=%x:%x},%s,%x\n", 2123 hpt, *(int *)hpt, (hpt->hpt_valid?"ok,":""), 2124 hpt->hpt_space, hpt->hpt_vpn << 9, 2125 buf, tlbptob(hpt->hpt_tlbpage)); 2126 } 2127 } 2128 #endif 2129 #endif 2130
Indexes created Sat Sep 20 22:09:52 GMT 2025