1 /* $NetBSD: pmap_pgmmu.c,v 1.1 2026/07/19 01:48:24 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2025, 2026 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Pmap module for the 68K Playground MMU, as found on the 68010-based 34 * Phaethon 1. 35 * 36 * The PGMMU is similar to the Sun3 MMU, but there are some differences, 37 * and they are not software-compatible. 38 * 39 * ==> Generally speaking, the MMU must be enabled for the system to 40 * function; if the MMU is disabled, all {User,Supervisor} {Data,Prog} 41 * cycles select the system ROM. This means not even the stack is 42 * accessible until the firmware sets up the MMU and enbles it, and 43 * the firmware runs in a virtual environment. When the kernel starts, 44 * it is running in the virtual environment set up by the firmware. 45 * 46 * ==> PGMMU has 64 contexts. Supervisor {Data,Prog} cycles are hard-wired 47 * to context 0. User {Data,Prog} cycles use the context specified by 48 * the Context Register (which, BTW, can be 0!). 49 * 50 * ==> Context 0's Segment Map has its own selector so that changing the 51 * kernel's Segment Map doesn't require setting the Context Register. 52 * 53 * ==> Each context has 512 32KB segments in the Segment Map. Each 54 * 16-bit Segment Map entry (SME) has a valid bit and a 15-bit 55 * number representing the Page Map Entry Group (PMEG) that maps 56 * the segment. 57 * 58 * ==> Each PMEG is comprised of 8 4KB pages, each with independent 59 * Page Map entries (PMEs). There are 32,768 total PMEGs shared 60 * by all processes in the system. 61 * 62 * ==> All PMEGs used by the kernel are static. The kernel pmap makes a 63 * best effort to use PMEGs already used by the system firmware. 64 * 65 * ==> There is sufficient SRAM in the MMU to fully map all 64 contexts. 66 * If there are more than 64 concurrent processes in the system 67 * (including the kernel), then user processes will have to compete 68 * with one another for contexts and PMEGs. 69 * 70 * ==> No systems with this MMU have a data cache, so cache management is 71 * not a concern. 72 */ 73 74 #include "opt_ddb.h" 75 #include "opt_kgdb.h" 76 77 #include <sys/cdefs.h> 78 __KERNEL_RCSID(0, "$NetBSD: pmap_pgmmu.c,v 1.1 2026/07/19 01:48:24 thorpej Exp $"); 79 80 #include <sys/param.h> 81 #include <sys/systm.h> 82 #include <sys/evcnt.h> 83 #include <sys/proc.h> 84 #include <sys/pool.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/kmem.h> 88 89 #include <machine/pcb.h> 90 91 #include <uvm/uvm.h> 92 #include <uvm/uvm_physseg.h> 93 94 /****************************** SERIALIZATION ********************************/ 95 96 /* 97 * XXX Would like to make these do something lightweight-ish in 98 * XXX DIAGNOSTIC kernels (and also make ASSERT_SLEEPABLE() trip 99 * XXX if we're in a critical section). 100 */ 101 102 #define PMAP_CRIT_ENTER(code) code 103 #define PMAP_CRIT_EXIT(code) code 104 #define PMAP_CRIT_ASSERT() __nothing 105 106 /**************************** MMU CONFIGURATION ******************************/ 107 108 #include <hb68k/pg68k/pgmmu.h> 109 110 __CTASSERT(PAGE_SIZE == PGMMU_PAGE_SIZE); 111 __CTASSERT(NBPG == PGMMU_PAGE_SIZE); 112 113 #ifdef __mc68010__ 114 #define KERNEL_MAX_ADDRESS (1U << 24) 115 #else 116 #error KERNEL_MAX_ADDRESS TBD 117 #endif 118 static vaddr_t kernel_virtual_start; 119 vaddr_t kernel_virtual_max = KERNEL_MAX_ADDRESS; 120 121 /***************************** INSTRUMENTATION *******************************/ 122 123 #ifdef PMAP_EVENT_COUNTERS 124 static struct evcnt pmap_ctx_alloc_static_ev = 125 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap ctx_alloc", "static"); 126 EVCNT_ATTACH_STATIC(pmap_ctx_alloc_static_ev); 127 128 static struct evcnt pmap_ctx_alloc_dynamic_ev = 129 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap ctx_alloc", "dynamic"); 130 EVCNT_ATTACH_STATIC(pmap_ctx_alloc_dynamic_ev); 131 132 static struct evcnt pmap_ctx_alloc_steal_ev = 133 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap ctx_alloc", "steal"); 134 EVCNT_ATTACH_STATIC(pmap_ctx_alloc_steal_ev); 135 136 static struct evcnt pmap_pv_alloc_wait_ev = 137 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_alloc", "wait"); 138 EVCNT_ATTACH_STATIC(pmap_pv_alloc_wait_ev); 139 140 static struct evcnt pmap_pv_alloc_nowait_ev = 141 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_alloc", "nowait"); 142 EVCNT_ATTACH_STATIC(pmap_pv_alloc_nowait_ev); 143 144 static struct evcnt pmap_pv_enter_called_ev = 145 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_enter", "called"); 146 EVCNT_ATTACH_STATIC(pmap_pv_enter_called_ev); 147 148 static struct evcnt pmap_pv_remove_called_ev = 149 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap pv_remove", "called"); 150 EVCNT_ATTACH_STATIC(pmap_pv_remove_called_ev); 151 152 static struct evcnt pmap_enter_nowait_ev = 153 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "nowait"); 154 EVCNT_ATTACH_STATIC(pmap_enter_nowait_ev); 155 156 static struct evcnt pmap_enter_yeswait_ev = 157 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "yeswait"); 158 EVCNT_ATTACH_STATIC(pmap_enter_yeswait_ev); 159 160 static struct evcnt pmap_enter_wire_change_ev = 161 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "wire change"); 162 EVCNT_ATTACH_STATIC(pmap_enter_wire_change_ev); 163 164 static struct evcnt pmap_enter_prot_change_ev = 165 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "prot change"); 166 EVCNT_ATTACH_STATIC(pmap_enter_prot_change_ev); 167 168 static struct evcnt pmap_enter_pa_change_ev = 169 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pa change"); 170 EVCNT_ATTACH_STATIC(pmap_enter_pa_change_ev); 171 172 static struct evcnt pmap_enter_pv_alloc_fail_ev = 173 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pv alloc failed"); 174 EVCNT_ATTACH_STATIC(pmap_enter_pv_alloc_fail_ev); 175 176 static struct evcnt pmap_enter_pv_recycle_ev = 177 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap enter", "pv recycle"); 178 EVCNT_ATTACH_STATIC(pmap_enter_pv_recycle_ev); 179 180 static struct evcnt pmap_prm_got_pg_ev = 181 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "got pg"); 182 static struct evcnt pmap_prm_lookup_pg_hit_ev = 183 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "lookup pg hit"); 184 static struct evcnt pmap_prm_lookup_pg_miss_ev = 185 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "pmap prm", "lookup pg miss"); 186 EVCNT_ATTACH_STATIC(pmap_prm_got_pg_ev); 187 EVCNT_ATTACH_STATIC(pmap_prm_lookup_pg_hit_ev); 188 EVCNT_ATTACH_STATIC(pmap_prm_lookup_pg_miss_ev); 189 190 #define pmap_evcnt(e) pmap_ ## e ## _ev.ev_count++ 191 #else 192 #define pmap_evcnt(e) __nothing 193 #endif 194 195 /************************** FORWARD DECLARATIONS *****************************/ 196 197 struct pmap_completion; 198 199 static void pmap_segment_retain(pmap_t, vaddr_t); 200 static void pmap_segment_release(pmap_t, vaddr_t, int); 201 202 static void pmap_remove_mapping(pmap_t, vaddr_t, unsigned int, 203 struct vm_page *pg, struct pmap_completion *); 204 static void pmap_remove_all_internal(pmap_t, struct pmap_completion *); 205 206 /***************************** PHYS <-> VM PAGE ******************************/ 207 208 static bool pmap_initialized_p; 209 210 static inline struct vm_page * 211 pmap_pa_to_pg(paddr_t pa) 212 { 213 if (__predict_true(pmap_initialized_p)) { 214 return PHYS_TO_VM_PAGE(pa); 215 } 216 return NULL; 217 } 218 219 static pm_entry_t pmap_changebit(struct vm_page *, pm_entry_t, pm_entry_t); 220 221 /*************************** RESOURCE MANAGEMENT *****************************/ 222 223 static struct pmap kernel_pmap_store; 224 struct pmap * const kernel_pmap_ptr = &kernel_pmap_store; 225 226 /* 227 * Avoid a memory load when doing comparisons against pmap_kernel() 228 * within this compilation unit. 229 */ 230 #undef pmap_kernel 231 #define pmap_kernel() (&kernel_pmap_store) 232 233 static struct pool pmap_pool; 234 static struct pool pmap_pv_pool; 235 236 #define PMAP_PV_LOWAT 16 237 238 static void 239 pmap_alloc_init(void) 240 { 241 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 242 PVH_ATTR_MASK + 1, /* align */ 243 0, /* ioff */ 244 0, /* flags */ 245 "pmappv", /* wchan */ 246 &pool_allocator_meta, /* palloc */ 247 IPL_VM); /* ipl */ 248 249 /* 250 * Set a low water mark on the pv_entry pool, so that we are 251 * more likely to have these around even in extreme memory 252 * starvation. 253 */ 254 pool_setlowat(&pmap_pv_pool, PMAP_PV_LOWAT); 255 256 pool_init(&pmap_pool, sizeof(struct pmap), 257 0, /* align */ 258 0, /* ioff */ 259 0, /* flags */ 260 "pmappl", /* wchan */ 261 &pool_allocator_kmem, /* palloc */ 262 IPL_NONE); /* ipl */ 263 } 264 265 static inline pmap_t 266 pmap_alloc(void) 267 { 268 pmap_t pmap = pool_get(&pmap_pool, PR_WAITOK); 269 memset(pmap, 0, sizeof(*pmap)); 270 return pmap; 271 } 272 273 static inline void 274 pmap_free(pmap_t pmap) 275 { 276 pool_put(&pmap_pool, pmap); 277 } 278 279 static struct pv_entry * 280 pmap_pv_alloc(bool nowait) 281 { 282 struct pv_entry *pv; 283 284 #ifdef PMAP_EVENT_COUNTERS 285 if (nowait) { 286 pmap_evcnt(pv_alloc_nowait); 287 } else { 288 pmap_evcnt(pv_alloc_wait); 289 } 290 #endif 291 292 pv = pool_get(&pmap_pv_pool, nowait ? PR_NOWAIT : 0); 293 if (__predict_true(pv != NULL)) { 294 KASSERT((((uintptr_t)pv) & PVH_ATTR_MASK) == 0); 295 } 296 return pv; 297 } 298 299 static void 300 pmap_pv_free(struct pv_entry *pv) 301 { 302 pool_put(&pmap_pv_pool, pv); 303 } 304 305 /* 306 * Whenever we need to free resources back to the system, we want to 307 * do it in a batch with any locks released. So, we have this around 308 * to collect the garbage, as needed. 309 */ 310 struct pmap_completion { 311 struct pv_entry *pc_pvlist; 312 }; 313 314 static inline void 315 pmap_completion_init(struct pmap_completion *pc) 316 { 317 pc->pc_pvlist = NULL; 318 } 319 320 static void 321 pmap_completion_fini(struct pmap_completion *pc) 322 { 323 struct pv_entry *pv; 324 325 while ((pv = pc->pc_pvlist) != NULL) { 326 pc->pc_pvlist = pv->pv_next; 327 pmap_pv_free(pv); 328 } 329 } 330 331 /* 332 * List of all user pmaps, used to identify potential marks for 333 * resource theft. This list is kept LRU-ordered by pmap_activate(). 334 */ 335 static TAILQ_HEAD(, pmap) pmap_all_user_pmaps; 336 337 /* 338 * pmap_find_victim: 339 * 340 * Identify a pmap we can steal some resources from. 341 */ 342 static pmap_t 343 pmap_find_victim(void) 344 { 345 pmap_t pm, best_victim = NULL; 346 347 /* 348 * Maybe not the best selection criteria, but: 349 * the least-recently-used pmap with the fewest 350 * number of wired mappings. 351 */ 352 353 TAILQ_FOREACH(pm, &pmap_all_user_pmaps, pm_list) { 354 if (pm->pm_busy) { 355 continue; 356 } 357 if (pm->pm_context == NULL) { 358 continue; 359 } 360 if (pm->pm_stats.wired_count == 0) { 361 best_victim = pm; 362 break; 363 } 364 if (best_victim == NULL || 365 best_victim->pm_stats.wired_count > 366 pm->pm_stats.wired_count) { 367 best_victim = pm; 368 } 369 } 370 KASSERT(best_victim != NULL); 371 KASSERT(best_victim->pm_context != NULL); 372 373 return best_victim; 374 } 375 376 /* 377 * Generic bitmap management, used for pmegs. 378 */ 379 struct pmap_bitmap { 380 uint32_t * const bitmap_words; 381 unsigned int const bitmap_nwords; 382 unsigned int bitmap_alloc_hint; 383 }; 384 385 #define PMAP_BITMAP_DECL(bm, bits) \ 386 __CTASSERT((bits) >= 32); \ 387 __CTASSERT(powerof2(bits)); \ 388 static uint32_t bm ## _bitmap_words[(bits) >> 5]; \ 389 static struct pmap_bitmap bm ## _bitmap = { \ 390 .bitmap_words = bm ## _bitmap_words, \ 391 .bitmap_nwords = (bits) >> 5, \ 392 }; 393 394 #define BITMAP_NEXT_WORD(bm, x) \ 395 (((x) + 1) & ((bm)->bitmap_nwords - 1)) 396 397 #define BITMAP_WORD(x) ((x) >> 5) 398 #define BITMAP_WORDOFFS(x) ((x) << 5) 399 #define BITMAP_BIT(x) (1U << ((x) & 31)) 400 401 static void 402 pmap_bitmap_init(struct pmap_bitmap *bm) 403 { 404 for (unsigned int i = 0; i < bm->bitmap_nwords; i++) { 405 bm->bitmap_words[i] = 0xffffffffU; 406 } 407 } 408 409 static void 410 pmap_bitmap_claim(struct pmap_bitmap *bm, unsigned int v) 411 { 412 KASSERT(v < BITMAP_WORDOFFS(bm->bitmap_nwords)); 413 bm->bitmap_words[BITMAP_WORD(v)] &= ~BITMAP_BIT(v); 414 } 415 416 static bool 417 pmap_bitmap_claimed_p(struct pmap_bitmap *bm, unsigned int v) 418 { 419 KASSERT(v < BITMAP_WORDOFFS(bm->bitmap_nwords)); 420 return !(bm->bitmap_words[BITMAP_WORD(v)] & BITMAP_BIT(v)); 421 } 422 423 static int 424 pmap_bitmap_alloc(struct pmap_bitmap *bm) 425 { 426 unsigned int i; 427 int v; 428 429 for (i = bm->bitmap_alloc_hint;; 430 i = BITMAP_NEXT_WORD(bm, i)) { 431 v = ffs(bm->bitmap_words[i]) - 1; 432 if (v != -1) { 433 bm->bitmap_alloc_hint = i; 434 bm->bitmap_words[i] &= ~BITMAP_BIT(v); 435 return BITMAP_WORDOFFS(i) | v; 436 } 437 if (BITMAP_NEXT_WORD(bm, i) == bm->bitmap_alloc_hint) { 438 break; 439 } 440 } 441 442 return -1; 443 } 444 445 static void 446 pmap_bitmap_free(struct pmap_bitmap *bm, unsigned int v) 447 { 448 KASSERT(v < BITMAP_WORDOFFS(bm->bitmap_nwords)); 449 bm->bitmap_words[BITMAP_WORD(v)] |= BITMAP_BIT(v); 450 bm->bitmap_alloc_hint = BITMAP_WORD(v); 451 } 452 453 /************************ SME MANIPULATION HELPERS ***************************/ 454 455 static inline sm_entry_t 456 pmap_getsme(pmap_t pmap, vaddr_t va) 457 { 458 if (pmap == pmap_kernel()) { 459 return pgmmu_getsme0(va); 460 } 461 KASSERT(pmap->pm_context != NULL); 462 KASSERT(pmap->pm_context->ctx_num == pgmmu_getcontext()); 463 return pgmmu_getsme(va); 464 } 465 466 static inline void 467 pmap_setsme(pmap_t pmap, vaddr_t va, sm_entry_t sme) 468 { 469 if (pmap == pmap_kernel()) { 470 pgmmu_setsme0(va, sme); 471 } else { 472 KASSERT(pmap->pm_context != NULL); 473 KASSERT(pmap->pm_context->ctx_num == pgmmu_getcontext()); 474 pgmmu_setsme(va, sme); 475 } 476 } 477 478 static inline uint16_t 479 sme_pmeg(sm_entry_t sme) 480 { 481 return sme & SME_PMEG; 482 } 483 484 static inline bool 485 sme_valid_p(sm_entry_t sme) 486 { 487 return !!(sme & SME_V); 488 } 489 490 /************************ PME MANIPULATION HELPERS ***************************/ 491 492 static inline paddr_t 493 pme_pa(pm_entry_t pme) 494 { 495 return pgmmu_ptob(pme & PME_PFN); 496 } 497 498 static inline bool 499 pme_valid_p(pm_entry_t pme) 500 { 501 return !!(pme & PME_V); 502 } 503 504 static inline bool 505 pme_wired_p(pm_entry_t pme) 506 { 507 return !!(pme & PME_WIRED); 508 } 509 510 static inline bool 511 pme_managed_p(pm_entry_t pme) 512 { 513 return !!(pme & PME_PVLIST); 514 } 515 516 static inline pm_entry_t 517 pme_change_prot(pm_entry_t opme, vm_prot_t prot) 518 { 519 return (opme & ~PME_W) | ((prot & UVM_PROT_WRITE) ? PME_W : 0); 520 } 521 522 static inline unsigned int 523 pme_index(uint16_t pmeg, vaddr_t va) 524 { 525 return (((uint32_t)pmeg) << PGMMU_PMEG_SHIFT) + 526 (pgmmu_btop(va) & PGMMU_PMEG_OFFSET); 527 } 528 529 static inline pm_entry_t 530 pmap_make_pme(pmap_t pmap, paddr_t pa, vm_prot_t prot, u_int flags) 531 { 532 pm_entry_t npme = PME_V | 533 (pmap == pmap_kernel() ? PME_K : 0) | 534 ((prot & UVM_PROT_WRITE) ? PME_W : 0) | 535 pgmmu_btop(pa); 536 if (flags & UVM_PROT_WRITE) { 537 npme |= PME_M | PME_R; 538 } else if (flags & (UVM_PROT_READ | UVM_PROT_EXEC)) { 539 npme |= PME_R; 540 } 541 if (flags & PMAP_WIRED) { 542 npme |= PME_WIRED; 543 } 544 545 return npme; 546 } 547 548 /* These helpers assume that all kernel segments have valid pmegs. */ 549 static pm_entry_t 550 pmap_getkpme(vaddr_t va) 551 { 552 sm_entry_t sme = pgmmu_getsme0(va); 553 return pgmmu_getpme(pme_index(sme_pmeg(sme), va)); 554 } 555 556 static void 557 pmap_setkpme(vaddr_t va, pm_entry_t pme) 558 { 559 sm_entry_t sme = pgmmu_getsme0(va); 560 pgmmu_setpme(pme_index(sme_pmeg(sme), va), pme); 561 } 562 563 /*************************** CONTEXT MANAGEMENT ******************************/ 564 565 /* 566 * This MMU can do 64 contexts, which, to be honest, for a 68010 is kind 567 * of a lot! We're going to just assume that if a context has to be stolen 568 * from another pmap, that the PMEGs are going to get slurped up, too. So, 569 * if a context has to get stolen, then we are going to forcefully remove 570 * the entire thing, segmap included; it can all be reconstructed from the 571 * VM map. We will TRY to honor pmaps with wired mappings, but ultimately, 572 * the resources have to be shared. 573 * 574 * The upshot of this is that there's really no reason to keep a software 575 * copy of the segmap because, realistically, it's not very likely that 576 * a context will get stolen from another pmap, and thus there is no need 577 * for us to be able to reload a context quickly. 578 * 579 * So, it's all just kept in the hardware. 580 */ 581 582 /* 583 * We have a static context for the kernel (the segrefs aren't used 584 * at all, but this makes the logic easier), and we statically allocate 585 * a handful of contexts for user pmaps as well. This ensures that 586 * there's at least a few around to steal at any given time. 587 */ 588 #define STATIC_CONTEXTS 9 589 static struct pmap_context static_contexts[STATIC_CONTEXTS]; 590 static struct pmap_context *context_freelist; 591 static unsigned int context_last; 592 593 static unsigned int pmap_current_context; 594 595 static unsigned int 596 pmap_swap_context(unsigned int ctx) 597 { 598 const unsigned int rv = pmap_current_context; 599 if (ctx != rv) { 600 pgmmu_setcontext(ctx); 601 pmap_current_context = ctx; 602 } 603 604 return rv; 605 } 606 607 static inline unsigned int 608 pmap_context_enter(pmap_t pmap) 609 { 610 struct pmap_context * const ctx = pmap->pm_context; 611 612 KASSERT(ctx != NULL); 613 return pmap_swap_context(ctx->ctx_num); 614 } 615 616 static inline void 617 pmap_context_exit(unsigned int saved_ctx) 618 { 619 (void) pmap_swap_context(saved_ctx); 620 } 621 622 static struct pmap_context * 623 pmap_context_new(bool nowait) 624 { 625 struct pmap_context *ctx = NULL; 626 unsigned int ctx_num; 627 628 ctx_num = context_last + 1; 629 if (__predict_false(ctx_num == PGMMU_NUM_CONTEXTS)) { 630 return NULL; 631 } 632 633 if (ctx_num < STATIC_CONTEXTS) { 634 ctx = &static_contexts[ctx_num]; 635 pmap_evcnt(ctx_alloc_static); 636 } else { 637 PMAP_CRIT_EXIT(); 638 ctx = kmem_zalloc(sizeof(*ctx), 639 nowait ? KM_NOSLEEP : KM_SLEEP); 640 PMAP_CRIT_ENTER(); 641 if (!nowait) { 642 ctx_num = context_last + 1; 643 if (__predict_false(ctx_num >= 644 PGMMU_NUM_CONTEXTS)) { 645 kmem_free(ctx, sizeof(*ctx)); 646 return NULL; 647 } 648 } 649 pmap_evcnt(ctx_alloc_dynamic); 650 } 651 652 ctx->ctx_num = context_last = ctx_num; 653 return ctx; 654 } 655 656 static void 657 pmap_context_alloc(pmap_t pmap, bool nowait, struct pmap_completion *pc) 658 { 659 struct pmap_context *ctx; 660 661 KASSERT(pmap != pmap_kernel()); 662 KASSERT(pmap->pm_context == NULL); 663 664 if (__predict_true((ctx = context_freelist) != NULL)) { 665 context_freelist = ctx->ctx_next; 666 memset(ctx->ctx_segrefs, 0, sizeof(ctx->ctx_segrefs)); 667 goto got_one; 668 } 669 670 if (__predict_true((ctx = pmap_context_new(true)) != NULL)) { 671 /* 672 * We may have blocked while allocating memory, in 673 * which case, another thread may have succeeded in 674 * nabbing a context for this pmap. If that's the 675 * case, then put the new one we just created onto 676 * the free list and proceed with the one we now 677 * find ourselves in possession of. 678 */ 679 if (__predict_false(pmap->pm_context != NULL)) { 680 ctx->ctx_next = context_freelist; 681 context_freelist = ctx; 682 return; 683 } 684 goto got_one; 685 } 686 687 /* 688 * We're going to have to steal a context from someone else. 689 */ 690 pmap_t victim = pmap_find_victim(); 691 692 ctx = victim->pm_context; 693 pmap_remove_all_internal(victim, pc); 694 victim->pm_context = NULL; 695 pmap_evcnt(ctx_alloc_steal); 696 got_one: 697 pmap->pm_context = ctx; 698 if (__predict_true(curproc != NULL && 699 pmap == curproc->p_vmspace->vm_map.pmap)) { 700 pmap_context_enter(pmap); 701 } 702 } 703 704 static void 705 pmap_context_free(pmap_t pmap) 706 { 707 struct pmap_context *ctx; 708 709 if (__predict_true((ctx = pmap->pm_context) != NULL)) { 710 pmap->pm_context = NULL; 711 ctx->ctx_next = context_freelist; 712 context_freelist = ctx; 713 } 714 } 715 716 /***************************** PMEG MANAGEMENT *******************************/ 717 718 /* 719 * This MMU has a lot of PMEGs, plenty to go around. We'll almost always 720 * be able to find a free one. Because of this, the victim selection for 721 * the situation where we need to steal one does not need to be particularly 722 * sophisticated. 723 */ 724 725 PMAP_BITMAP_DECL(pmeg, PGMMU_NUM_PMEGS) 726 727 static unsigned int 728 pmap_pmeg_alloc(pmap_t pmap, struct pmap_completion *pc) 729 { 730 vaddr_t va, nextva; 731 sm_entry_t sme; 732 unsigned int seg, i, saved_ctx; 733 int pmeg; 734 735 pmeg = pmap_bitmap_alloc(&pmeg_bitmap); 736 if (pmeg != -1) { 737 return pmeg; 738 } 739 740 /* 741 * We're going to have to steal a pmeg from someone else. 742 */ 743 pmap_t victim = pmap_find_victim(); 744 745 /* 746 * Just steal the pmeg of the first valid segment we find. 747 */ 748 saved_ctx = pmap_context_enter(victim); 749 for (seg = 0, va = 0; seg < PGMMU_NUM_SEGS; seg++, va = nextva) { 750 nextva = va + PGMMU_SEG_SIZE; 751 sme = pgmmu_getsme(va); 752 if (sme_valid_p(sme)) { 753 pmap_segment_retain(victim, va); 754 pmeg = sme_pmeg(sme); 755 for (i = 0; i < PGMMU_PMEG_SIZE; i++) { 756 pmap_remove_mapping(victim, 757 va + (i * PAGE_SIZE), pmeg, NULL, pc); 758 } 759 pmap_segment_release(victim, va, -1); 760 break; 761 } 762 /* 763 * We are guaranteed to find something because the victim 764 * will have something to steal. If we go past the max 765 * user address or roll over back to 0, then we're well and 766 * truly <fill in the blank>. 767 */ 768 KASSERT(nextva < VM_MAXUSER_ADDRESS); 769 KASSERT(nextva != 0); 770 } 771 pmap_context_exit(saved_ctx); 772 KASSERT(pmeg != -1); 773 774 return pmeg; 775 } 776 777 static inline void 778 pmap_pmeg_free(unsigned int pmeg) 779 { 780 pmap_bitmap_free(&pmeg_bitmap, pmeg); 781 } 782 783 static void 784 pmap_segment_retain(pmap_t pmap, vaddr_t va) 785 { 786 const unsigned int seg = pgmmu_btos(va); 787 struct pmap_context * const ctx = pmap->pm_context; 788 789 if (__predict_true(pmap != pmap_kernel())) { 790 KASSERT(ctx != NULL); 791 ctx->ctx_segrefs[seg]++; 792 KASSERT(ctx->ctx_segrefs[seg] != 0); 793 } 794 } 795 796 static void 797 pmap_segment_release(pmap_t pmap, vaddr_t va, int pmeg) 798 { 799 const unsigned int seg = pgmmu_btos(va); 800 struct pmap_context * const ctx = pmap->pm_context; 801 802 if (__predict_true(pmap != pmap_kernel())) { 803 KASSERT(ctx != NULL); 804 KASSERT(ctx->ctx_segrefs[seg] != 0); 805 if (--ctx->ctx_segrefs[seg] == 0 && pmeg != -1) { 806 pmap_setsme(pmap, va, 0); 807 pmap_pmeg_free(pmeg); 808 } 809 } 810 } 811 812 /************************** P->V ENTRY MANAGEMENT ****************************/ 813 814 /* 815 * pmap_pv_enter: 816 * 817 * Add a physical->virtual entry to the pv table. Caller must provide 818 * the storage for the new PV entry. 819 */ 820 static void 821 pmap_pv_enter(pmap_t pmap, struct vm_page *pg, vaddr_t va, 822 unsigned int pmeg, struct pv_entry *newpv) 823 { 824 pmap_evcnt(pv_enter_called); 825 826 PMAP_CRIT_ASSERT(); 827 KASSERT(newpv != NULL); 828 829 newpv->pv_pmap = pmap; 830 newpv->pv_vf = va; 831 newpv->pv_pmeg = pmeg; 832 newpv->pv_next = VM_MDPAGE_PVS(pg); 833 VM_MDPAGE_SETPVP(VM_MDPAGE_HEAD_PVP(pg), newpv); 834 } 835 836 /* 837 * pmap_pv_remove: 838 * 839 * Remove a physical->virtual entry from the pv table. 840 */ 841 static void 842 pmap_pv_remove(pmap_t pmap, struct vm_page *pg, vaddr_t va, 843 struct pmap_completion *pc) 844 { 845 struct pv_entry **pvp, *pv; 846 847 pmap_evcnt(pv_remove_called); 848 849 PMAP_CRIT_ASSERT(); 850 851 for (pvp = VM_MDPAGE_HEAD_PVP(pg), pv = VM_MDPAGE_PVS(pg); 852 pv != NULL; 853 pvp = &pv->pv_next, pv = *pvp) { 854 if (pmap == pv->pv_pmap && va == PV_VA(pv)) { 855 break; 856 } 857 } 858 859 KASSERT(pv != NULL); 860 VM_MDPAGE_SETPVP(pvp, pv->pv_next); 861 862 KASSERT(pc != NULL); 863 pv->pv_next = pc->pc_pvlist; 864 pc->pc_pvlist = pv; 865 } 866 867 /***************** PMAP INTERFACE (AND ADJACENT) FUNCTIONS *******************/ 868 869 static inline void 870 pmap_stat_update_impl(long *valp, int val) 871 { 872 *valp += val; 873 } 874 875 #define pmap_stat_update(pm, stat, delta) \ 876 pmap_stat_update_impl(&(pm)->pm_stats.stat, (delta)) 877 878 static inline void 879 pmap_stat_set_impl(long *valp, int val) 880 { 881 atomic_store_relaxed(valp, val); 882 } 883 884 #define pmap_stat_set(pm, stat, val) \ 885 pmap_stat_set_impl(&(pm)->pm_stats.stat, (val)) 886 887 /* 888 * pmap_pinit: 889 * 890 * Common bits of pmap structure initialization shared between 891 * the kernel pmap and user pmaps. 892 */ 893 static void 894 pmap_pinit(pmap_t pmap, struct pmap_context *ctx) 895 { 896 pmap->pm_context = ctx; 897 atomic_store_relaxed(&pmap->pm_refcnt, 1); 898 } 899 900 /* 901 * pmap_virtual_space: [ INTERFACE ] 902 * 903 * Define the initial bounds of the kernel virtual address space. 904 * 905 * In this implementation, the start address we return marks the 906 * end of the statically allocated special kernel virtual addresses 907 * set up in pmap_bootstrap1(). And since we have fixed mapping 908 * resources and thus don't need to have a pmap_growkernel(), we 909 * return the fill limit right away (clamped by whatever top-of 910 * address-space mappings that we need to keep around, like firmware 911 * and device mappings). 912 */ 913 void 914 pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp) 915 { 916 *vstartp = kernel_virtual_start; 917 *vendp = kernel_virtual_max; 918 } 919 920 /* 921 * pmap_init: [ INTERFACE ] 922 * 923 * Initialize the pmap module. Called by vm_init(), to initialize any 924 * structures that the pmap system needs to map virtual memory. 925 */ 926 void 927 pmap_init(void) 928 { 929 /* Initialize the pmap / pv_entry allocators. */ 930 pmap_alloc_init(); 931 932 /* Now it's safe to do P->V entry recording! */ 933 pmap_initialized_p = true; 934 } 935 936 /* 937 * pmap_create: [ INTERFACE ] 938 * 939 * Create and return a physical map. 940 */ 941 pmap_t 942 pmap_create(void) 943 { 944 pmap_t pmap; 945 946 /* 947 * We don't allocate a context until the first mapping is 948 * entered. 949 */ 950 pmap = pmap_alloc(); 951 pmap_pinit(pmap, NULL); 952 953 PMAP_CRIT_ENTER(); 954 TAILQ_INSERT_TAIL(&pmap_all_user_pmaps, pmap, pm_list); 955 PMAP_CRIT_EXIT(); 956 957 return pmap; 958 } 959 960 /* 961 * pmap_destroy: [ INTERFACE ] 962 * 963 * Drop the reference count on the specified pmap, releasing 964 * all resources if the reference count drops to zero. 965 */ 966 void 967 pmap_destroy(pmap_t pmap) 968 { 969 unsigned int newval; 970 971 PMAP_CRIT_ENTER(); 972 KASSERT(pmap->pm_refcnt > 0); 973 newval = --pmap->pm_refcnt; 974 975 if (newval) { 976 PMAP_CRIT_EXIT(); 977 return; 978 } 979 980 /* We assume all mappings have been removed. */ 981 KASSERT(pmap->pm_stats.resident_count == 0); 982 if (pmap->pm_context != NULL) { 983 pmap_context_free(pmap); 984 } 985 986 TAILQ_REMOVE(&pmap_all_user_pmaps, pmap, pm_list); 987 988 PMAP_CRIT_EXIT(); 989 990 pmap_free(pmap); 991 } 992 993 /* 994 * pmap_reference: [ INTERFACE ] 995 * 996 * Add a reference to the specified pmap. 997 */ 998 void 999 pmap_reference(pmap_t pmap) 1000 { 1001 PMAP_CRIT_ENTER(); 1002 pmap->pm_refcnt++; 1003 KASSERT(pmap->pm_refcnt > 0); 1004 PMAP_CRIT_EXIT(); 1005 } 1006 1007 /* 1008 * pmap_remove_mapping: 1009 * 1010 * Invalidate a single page denoted by pmap/va. 1011 */ 1012 static void 1013 pmap_remove_mapping(pmap_t pmap, vaddr_t va, unsigned int pmeg, 1014 struct vm_page *pg, struct pmap_completion *pc) 1015 { 1016 const unsigned int pmeidx = pme_index(pmeg, va); 1017 const pm_entry_t opme = pgmmu_getpme(pmeidx); 1018 1019 if (! pme_valid_p(opme)) { 1020 return; 1021 } 1022 1023 const paddr_t pa = pme_pa(opme); 1024 KASSERT(pg == NULL || pa == VM_PAGE_TO_PHYS(pg)); 1025 1026 /* Update statistics. */ 1027 if (pme_wired_p(opme)) { 1028 pmap_stat_update(pmap, wired_count, -1); 1029 } 1030 pmap_stat_update(pmap, resident_count, -1); 1031 1032 if (__predict_true(pg == NULL)) { 1033 pg = pmap_pa_to_pg(pa); 1034 if (pg != NULL) { 1035 pmap_evcnt(prm_lookup_pg_hit); 1036 } else { 1037 pmap_evcnt(prm_lookup_pg_miss); 1038 } 1039 } else { 1040 pmap_evcnt(prm_got_pg); 1041 } 1042 if (__predict_true(pg != NULL)) { 1043 KASSERT(pme_managed_p(opme)); 1044 /* Update cached M/R bits from mapping that's going away. */ 1045 VM_MDPAGE_ADD_MR(pg, opme); 1046 pmap_pv_remove(pmap, pg, va, pc); 1047 } else { 1048 KASSERT(! pme_managed_p(opme)); 1049 } 1050 1051 /* Zap the Page Map entry. */ 1052 pgmmu_setpme(pmeidx, 0); 1053 pmap_segment_release(pmap, va, pmeg); 1054 } 1055 1056 /* 1057 * pmap_remove: [ INTERFACE ] 1058 * 1059 * Remove the given range of addresses from the specified map. 1060 * 1061 * It is assumed that the start and end are properly rounded 1062 * to the page size. 1063 * 1064 * N.B. Callers of pmap_remove_internal() are expected to 1065 * provide an initialized completion context, which we 1066 * will finalize. 1067 */ 1068 static void 1069 pmap_remove_internal(pmap_t pmap, vaddr_t sva, vaddr_t eva, 1070 struct pmap_completion *pc) 1071 { 1072 sm_entry_t sme; 1073 vaddr_t nextseg; 1074 unsigned int saved_ctx; 1075 unsigned int pmeg; 1076 1077 PMAP_CRIT_ENTER(pmap->pm_busy++); 1078 1079 if (pmap->pm_context == NULL) { 1080 KASSERT(pmap->pm_stats.resident_count == 0); 1081 goto out; 1082 } 1083 saved_ctx = pmap_context_enter(pmap); 1084 1085 while (sva < eva) { 1086 nextseg = pgmmu_next_seg(sva); 1087 if (nextseg == 0 || nextseg > eva) { 1088 nextseg = eva; 1089 } 1090 1091 sme = pmap_getsme(pmap, sva); 1092 if (! sme_valid_p(sme)) { 1093 /* 1094 * No PMEG for this segment; advance to the 1095 * next one. 1096 */ 1097 sva = nextseg; 1098 continue; 1099 } 1100 pmeg = sme_pmeg(sme); 1101 1102 for (; sva < nextseg; sva += PAGE_SIZE) { 1103 pmap_remove_mapping(pmap, sva, pmeg, NULL, pc); 1104 } 1105 } 1106 1107 pmap_context_exit(saved_ctx); 1108 out: 1109 PMAP_CRIT_EXIT(pmap->pm_busy--); 1110 pmap_completion_fini(pc); 1111 } 1112 1113 void 1114 pmap_remove(pmap_t pmap, vaddr_t sva, vaddr_t eva) 1115 { 1116 struct pmap_completion pc; 1117 pmap_completion_init(&pc); 1118 pmap_remove_internal(pmap, sva, eva, &pc); 1119 /* pmap_remove_internal() calls pmap_completion_fini(). */ 1120 } 1121 1122 /* 1123 * pmap_remove_all: [ INTERFACE ] 1124 * 1125 * Remove all mappings from a pmap in bulk. This is only called 1126 * when it's known that the address space is no longer visible to 1127 * any user process (e.g. during exit or exec). 1128 */ 1129 static void 1130 pmap_remove_all_internal(pmap_t pmap, struct pmap_completion *pc) 1131 { 1132 unsigned int pmeg, seg, i, saved_ctx; 1133 vaddr_t va, nextva; 1134 sm_entry_t sme; 1135 1136 saved_ctx = pmap_context_enter(pmap); 1137 for (seg = 0, va = 0; seg < PGMMU_NUM_SEGS; seg++, va = nextva) { 1138 nextva = va + PGMMU_SEG_SIZE; 1139 sme = pgmmu_getsme(va); 1140 if (sme_valid_p(sme)) { 1141 pmeg = sme_pmeg(sme); 1142 for (i = 0; i < PGMMU_PMEG_SIZE; 1143 i++, va += PAGE_SIZE) { 1144 pmap_remove_mapping(pmap, va, pmeg, NULL, pc); 1145 } 1146 } 1147 } 1148 pmap_context_exit(saved_ctx); 1149 } 1150 1151 bool 1152 pmap_remove_all(pmap_t pmap) 1153 { 1154 struct pmap_completion pc; 1155 1156 KASSERT(pmap != pmap_kernel()); 1157 1158 pmap_completion_init(&pc); 1159 1160 PMAP_CRIT_ENTER(pmap->pm_busy++); 1161 if (pmap->pm_context != NULL) { 1162 pmap_remove_all_internal(pmap, &pc); 1163 } 1164 PMAP_CRIT_EXIT(pmap->pm_busy--); 1165 1166 pmap_completion_fini(&pc); 1167 1168 return true; 1169 } 1170 1171 /* 1172 * pmap_page_protect: [ INTERFACE ] 1173 * 1174 * Lower the permission for all mappings to a given page to 1175 * the permissions specified. 1176 */ 1177 void 1178 pmap_page_protect(struct vm_page *pg, vm_prot_t prot) 1179 { 1180 struct pmap_completion pc; 1181 struct pv_entry *pv; 1182 1183 if (prot & UVM_PROT_WRITE) { 1184 /* No protection to revoke. */ 1185 return; 1186 } 1187 1188 if (prot & UVM_PROT_READ) { 1189 /* Making page copy-on-write. */ 1190 pmap_changebit(pg, 0, (pm_entry_t)~PME_W); 1191 return; 1192 } 1193 1194 /* Removing all mappings for a page. */ 1195 pmap_completion_init(&pc); 1196 1197 PMAP_CRIT_ENTER(); 1198 1199 unsigned int saved_ctx = pmap_current_context; 1200 1201 while ((pv = VM_MDPAGE_PVS(pg)) != NULL) { 1202 pmap_context_enter(pv->pv_pmap); 1203 pv->pv_pmap->pm_busy++; 1204 pmap_remove_mapping(pv->pv_pmap, PV_VA(pv), pv->pv_pmeg, 1205 pg, &pc); 1206 pv->pv_pmap->pm_busy--; 1207 } 1208 1209 pmap_context_exit(saved_ctx); 1210 1211 PMAP_CRIT_EXIT(); 1212 1213 pmap_completion_fini(&pc); 1214 } 1215 1216 /* 1217 * pmap_protect: [ INTERFACE ] 1218 * 1219 * Set the physical protection on the specified range of this map 1220 * as requested. 1221 */ 1222 void 1223 pmap_protect(pmap_t pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) 1224 { 1225 sm_entry_t sme; 1226 pm_entry_t opme, npme; 1227 vaddr_t nextseg; 1228 unsigned int saved_ctx; 1229 unsigned int pmeg; 1230 unsigned int pmeidx; 1231 1232 if ((prot & UVM_PROT_READ) == 0) { 1233 struct pmap_completion pc; 1234 pmap_completion_init(&pc); 1235 pmap_remove_internal(pmap, sva, eva, &pc); 1236 /* pmap_remove_internal() calls pmap_completion_fini(). */ 1237 return; 1238 } 1239 1240 PMAP_CRIT_ENTER(pmap->pm_busy++); 1241 1242 if (pmap->pm_context == NULL) { 1243 KASSERT(pmap->pm_stats.resident_count == 0); 1244 goto out; 1245 } 1246 saved_ctx = pmap_context_enter(pmap); 1247 1248 while (sva < eva) { 1249 nextseg = pgmmu_next_seg(sva); 1250 if (nextseg == 0 || nextseg > eva) { 1251 nextseg = eva; 1252 } 1253 1254 sme = pmap_getsme(pmap, sva); 1255 if (! sme_valid_p(sme)) { 1256 /* 1257 * No PMEG for this segment; advance to the 1258 * next one. 1259 */ 1260 sva = nextseg; 1261 continue; 1262 } 1263 pmeg = sme_pmeg(sme); 1264 1265 /* 1266 * Change protection on mapping if it is valid and doesn't 1267 * already have the correct protection. 1268 */ 1269 for (pmeidx = pme_index(pmeg, sva); 1270 sva < nextseg; pmeidx++, sva += PAGE_SIZE) { 1271 opme = pgmmu_getpme(pmeidx); 1272 if (! pme_valid_p(opme)) { 1273 continue; 1274 } 1275 npme = pme_change_prot(opme, prot); 1276 if (npme == opme) { 1277 continue; 1278 } 1279 pgmmu_setpme(pmeidx, npme); 1280 } 1281 } 1282 pmap_context_exit(saved_ctx); 1283 out: 1284 PMAP_CRIT_EXIT(pmap->pm_busy--); 1285 } 1286 1287 /* 1288 * pmap_enter: [ INTERFACE ] 1289 * 1290 * Insert the given physical address (pa) at the specified 1291 * virtual address (va) in the target physical map with the 1292 * protection requested. 1293 * 1294 * If specified, the page will be wired down, meaning that 1295 * related pme can not be reclaimed. 1296 * 1297 * Note: This is the only routine which MAY NOT lazy-evaluate 1298 * or lose information. That is, this routine must actually 1299 * insert this page into the given map NOW. 1300 */ 1301 int 1302 pmap_enter(pmap_t pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1303 { 1304 pm_entry_t npme, opme; 1305 sm_entry_t sme; 1306 unsigned int pmeg; 1307 unsigned int pmeidx; 1308 unsigned int saved_ctx; 1309 struct pv_entry *newpv; 1310 struct pmap_completion pc; 1311 int error = 0; 1312 const bool nowait = !!(flags & PMAP_CANFAIL); 1313 1314 pmap_completion_init(&pc); 1315 1316 struct vm_page * const pg = pmap_pa_to_pg(pa); 1317 1318 PMAP_CRIT_ENTER(pmap->pm_busy++); 1319 1320 if (nowait) { 1321 pmap_evcnt(enter_nowait); 1322 } else { 1323 pmap_evcnt(enter_yeswait); 1324 } 1325 1326 /* If we don't already have a context, get one. */ 1327 if (__predict_false(pmap->pm_context == NULL)) { 1328 pmap_context_alloc(pmap, nowait, &pc); 1329 } 1330 1331 saved_ctx = pmap_context_enter(pmap); 1332 1333 /* Check to see if we already have a valid PMEG for this mapping. */ 1334 sme = pmap_getsme(pmap, va); 1335 if (sme_valid_p(sme)) { 1336 /* Yup! */ 1337 pmeg = sme_pmeg(sme); 1338 } else { 1339 /* Need to allocate one. */ 1340 pmeg = pmap_pmeg_alloc(pmap, &pc); 1341 pmap_setsme(pmap, va, SME_V | pmeg); 1342 } 1343 pmap_segment_retain(pmap, va); 1344 1345 pmeidx = pme_index(pmeg, va); 1346 1347 /* Compute the new PME. */ 1348 npme = pmap_make_pme(pmap, pa, prot, flags); 1349 1350 /* Fetch the old PME. */ 1351 opme = pgmmu_getpme(pmeidx); 1352 1353 /* 1354 * Check to see if there is an old mapping at this address. 1355 * It might simply be a wiring or protection change. 1356 */ 1357 if (pme_valid_p(opme)) { 1358 restart: 1359 if (pme_pa(opme) == pa) { 1360 /* 1361 * Just a protection or wiring change. 1362 * 1363 * Since the old PME is handy, go ahead and update 1364 * the cached M/R attributes now. Normally we would 1365 * do this in pmap_remove_mapping(), but we're not 1366 * taking that path in this case. We also add in 1367 * any M/R attributes hinted by the access type 1368 * that brought us to pmap_enter() in the first 1369 * place (a write-fault on a writable page mapped 1370 * read-only during a page-out, for example). 1371 * 1372 * Also ensure that the PV list status of the mapping 1373 * is consistent. 1374 */ 1375 if (__predict_true(pg != NULL)) { 1376 VM_MDPAGE_ADD_MR(pg, opme | npme); 1377 KASSERT(pme_managed_p(opme)); 1378 npme |= PME_PVLIST; 1379 } 1380 1381 /* Set the new PME. */ 1382 pgmmu_setpme(pmeidx, npme); 1383 1384 const pm_entry_t diff = opme ^ npme; 1385 1386 #ifdef PMAP_EVENT_COUNTERS 1387 if (diff & PME_WIRED) { 1388 pmap_evcnt(enter_wire_change); 1389 } 1390 if (diff & PME_W) { 1391 pmap_evcnt(enter_prot_change); 1392 } 1393 #endif 1394 1395 if (pme_wired_p(diff)) { 1396 pmap_stat_update(pmap, wired_count, 1397 pme_wired_p(npme) ? 1 : -1); 1398 } 1399 1400 /* All done! */ 1401 goto out_release; 1402 } 1403 1404 /* 1405 * The mapping has completely changed. Need to remove 1406 * the old one first. 1407 * 1408 * This will drop the retain count on the segment owned 1409 * by the previous mapping, but the newly-entered mapping 1410 * will inherit the retain count taken when we validated 1411 * the SME. 1412 */ 1413 pmap_evcnt(enter_pa_change); 1414 pmap_remove_mapping(pmap, va, pmeg, NULL, &pc); 1415 } 1416 1417 /* Update pmap stats now. */ 1418 pmap_stat_update(pmap, resident_count, 1); 1419 if (__predict_false(pme_wired_p(npme))) { 1420 pmap_stat_update(pmap, wired_count, 1); 1421 } 1422 1423 if (__predict_true(pg != NULL)) { 1424 /* 1425 * Managed pages also go on the PV list, so we are 1426 * going to need a PV entry. 1427 */ 1428 newpv = pc.pc_pvlist; 1429 if (__predict_true(newpv == NULL)) { 1430 /* 1431 * No PV entry to recycle; allocate a new one. 1432 * Because this is an extremely common case, we 1433 * are first going to attempt allocation while 1434 * still in the critical section. If that fails 1435 * and waiting is allowed, we'll leave the critical 1436 * section and try a blocking allocation. 1437 */ 1438 newpv = pmap_pv_alloc(true/*nowait flag*/); 1439 if (__predict_false(newpv == NULL)) { 1440 if (nowait) { 1441 pmap_evcnt(enter_pv_alloc_fail); 1442 error = ENOMEM; 1443 goto out_release; 1444 } 1445 /* XXX Should steal a PV */ 1446 PMAP_CRIT_EXIT(); 1447 newpv = pmap_pv_alloc(false/*nowait flag*/); 1448 KASSERT(newpv != NULL); 1449 PMAP_CRIT_ENTER(); 1450 /* 1451 * Because we may have blocked while allocating 1452 * the PV entry, we have to re-validate our 1453 * environment, as another thread could have 1454 * inserted a mapping here behind our back. 1455 */ 1456 opme = pgmmu_getpme(pmeidx); 1457 if (__predict_false(pme_valid_p(opme))) { 1458 pmap_stat_update(pmap, 1459 resident_count, -1); 1460 if (pme_wired_p(npme)) { 1461 pmap_stat_update(pmap, 1462 wired_count, -1); 1463 } 1464 newpv->pv_next = pc.pc_pvlist; 1465 pc.pc_pvlist = newpv; 1466 goto restart; 1467 } 1468 } 1469 } else { 1470 pmap_evcnt(enter_pv_recycle); 1471 pc.pc_pvlist = newpv->pv_next; 1472 newpv->pv_next = NULL; 1473 } 1474 1475 /* Enter the mapping into the PV list. */ 1476 pmap_pv_enter(pmap, pg, va, pmeg, newpv); 1477 npme |= PME_PVLIST; 1478 1479 /* ...and seed the page attributes. */ 1480 VM_MDPAGE_ADD_MR(pg, npme); 1481 } 1482 1483 /* 1484 * Set the new PME. The new mapping takes ownership of the segment 1485 * retain count we took earlier. 1486 */ 1487 pgmmu_setpme(pmeidx, npme); 1488 goto out_crit_exit; 1489 1490 out_release: 1491 pmap_segment_release(pmap, va, pmeg); 1492 out_crit_exit: 1493 pmap_context_exit(saved_ctx); 1494 PMAP_CRIT_EXIT(pmap->pm_busy--); 1495 1496 pmap_completion_fini(&pc); 1497 return error; 1498 } 1499 1500 /* 1501 * pmap_kenter_pa: [ INTERFACE ] 1502 * 1503 * Enter a va -> pa mapping into the kernel pmap without any 1504 * physical->virtual tracking. 1505 */ 1506 void 1507 pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot, u_int flags) 1508 { 1509 pmap_t const pmap = pmap_kernel(); 1510 1511 const sm_entry_t sme = pgmmu_getsme0(va); 1512 1513 /* The kernel context is fully loaded with PMEGs. */ 1514 KASSERT(sme_valid_p(sme)); 1515 const unsigned int pmeidx = pme_index(sme_pmeg(sme), va); 1516 1517 /* Build the new PME. */ 1518 const pm_entry_t npme = 1519 pmap_make_pme(pmap, pa, prot, flags | PMAP_WIRED); 1520 1521 /* There must not be a valid PTE here. */ 1522 KASSERT(! pme_valid_p(pgmmu_getpme(pmeidx))); 1523 1524 /* Set the new PME. */ 1525 pgmmu_setpme(pmeidx, npme); 1526 1527 pmap_stat_update(pmap, resident_count, 1); 1528 pmap_stat_update(pmap, wired_count, 1); 1529 } 1530 1531 /* 1532 * pmap_kremove: [ INTERFACE ] 1533 * 1534 * Remove a mapping entered with pmap_kenter_pa() starting at va, 1535 * for size bytes (assumed to be page rounded). 1536 */ 1537 void 1538 pmap_kremove(vaddr_t va, vsize_t size) 1539 { 1540 int count = 0; 1541 sm_entry_t sme; 1542 pm_entry_t opme; 1543 unsigned int pmeidx; 1544 unsigned int pmeg; 1545 vaddr_t eva = va + size; 1546 vaddr_t nextseg; 1547 1548 while (va < eva) { 1549 nextseg = pgmmu_next_seg(va); 1550 if (nextseg == 0 || nextseg > eva) { 1551 nextseg = eva; 1552 } 1553 1554 sme = pgmmu_getsme0(va); 1555 KASSERT(sme_valid_p(sme)); 1556 1557 pmeg = sme_pmeg(sme); 1558 1559 for (pmeidx = pme_index(pmeg, va); 1560 va < nextseg; pmeidx++, va += PAGE_SIZE) { 1561 opme = pgmmu_getpme(pmeidx); 1562 if (pme_valid_p(opme)) { 1563 KASSERT(! pme_managed_p(opme)); 1564 KASSERT(pme_wired_p(opme)); 1565 /* Zap the mapping. */ 1566 pgmmu_setpme(pmeidx, 0); 1567 count++; 1568 } 1569 } 1570 } 1571 1572 /* Update stats. */ 1573 if (__predict_true(count != 0)) { 1574 pmap_stat_update(pmap_kernel(), resident_count, -count); 1575 pmap_stat_update(pmap_kernel(), wired_count, -count); 1576 } 1577 } 1578 1579 /* 1580 * pmap_unwire: [ INTERFACE ] 1581 * 1582 * Clear the wired attribute for a map/virtual-address pair. 1583 * 1584 * The mapping must already exist in the pmap. 1585 * (Except, it might not if we stole it.) 1586 */ 1587 void 1588 pmap_unwire(pmap_t pmap, vaddr_t va) 1589 { 1590 PMAP_CRIT_ENTER(pmap->pm_busy++); 1591 1592 if (pmap->pm_context == NULL) { 1593 KASSERT(pmap->pm_stats.resident_count == 0); 1594 goto out; 1595 } 1596 const unsigned int saved_ctx = pmap_context_enter(pmap); 1597 1598 const sm_entry_t sme = pmap_getsme(pmap, va); 1599 if (sme_valid_p(sme)) { 1600 const unsigned int pmeidx = pme_index(sme_pmeg(sme), va); 1601 const pm_entry_t pme = pgmmu_getpme(pmeidx); 1602 if (pme_valid_p(pme) && pme_wired_p(pme)) { 1603 pgmmu_setpme(pmeidx, pme & ~PME_WIRED); 1604 pmap_stat_update(pmap, wired_count, -1); 1605 } 1606 } 1607 1608 pmap_context_exit(saved_ctx); 1609 out: 1610 PMAP_CRIT_EXIT(pmap->pm_busy--); 1611 } 1612 1613 /* 1614 * pmap_extract: [ INTERFACE ] 1615 * 1616 * Extract the physical address associated with the given 1617 * pmap/virtual address pair. 1618 * 1619 * pmap_extract_info: 1620 * 1621 * Like pmap_extract(), but also returns information 1622 * about the mapping (wired, cache-inhibited, etc.) 1623 */ 1624 bool 1625 pmap_extract_info(pmap_t pmap, vaddr_t va, paddr_t *pap, int *flagsp) 1626 { 1627 unsigned int saved_ctx; 1628 bool rv = false; 1629 1630 PMAP_CRIT_ENTER(pmap->pm_busy++); 1631 if (pmap->pm_context == NULL) { 1632 KASSERT(pmap->pm_stats.resident_count == 0); 1633 goto out; 1634 } 1635 saved_ctx = pmap_context_enter(pmap); 1636 1637 const sm_entry_t sme = pmap_getsme(pmap, va); 1638 if (__predict_true(sme_valid_p(sme))) { 1639 const unsigned int pmeidx = pme_index(sme_pmeg(sme), va); 1640 const pm_entry_t pme = pgmmu_getpme(pmeidx); 1641 if (__predict_true(pme_valid_p(pme))) { 1642 if (__predict_true(pap != NULL)) { 1643 *pap = pme_pa(pme) | (va & PGOFSET); 1644 } 1645 if (__predict_false(flagsp != NULL)) { 1646 /* 1647 * No systems with this MMU have a data 1648 * cache, so always indicate that the 1649 * mappings are not cached. 1650 */ 1651 *flagsp = PMAP_NOCACHE | 1652 (pme_wired_p(pme) ? PMAP_WIRED : 0); 1653 } 1654 rv = true; 1655 } 1656 } 1657 1658 pmap_context_exit(saved_ctx); 1659 out: 1660 PMAP_CRIT_EXIT(pmap->pm_busy--); 1661 return rv; 1662 } 1663 1664 bool 1665 pmap_extract(pmap_t pmap, vaddr_t va, paddr_t *pap) 1666 { 1667 return pmap_extract_info(pmap, va, pap, NULL); 1668 } 1669 1670 /* 1671 * vtophys: 1672 * 1673 * Dumber version of pmap_extract(pmap_kernel(), ...) 1674 */ 1675 paddr_t 1676 vtophys(vaddr_t va) 1677 { 1678 paddr_t pa; 1679 bool rv __diagused; 1680 1681 rv = pmap_extract_info(pmap_kernel(), va, &pa, NULL); 1682 KASSERT(rv); 1683 return rv ? pa : -1; 1684 } 1685 1686 /* 1687 * kvtop: 1688 * 1689 * Sigh. 1690 */ 1691 int 1692 kvtop(void *v) 1693 { 1694 return (int)vtophys((vaddr_t)v); 1695 } 1696 1697 /* 1698 * pmap_copy: [ INTERFACE ] 1699 * 1700 * Copy the mapping range specified by src_addr/len 1701 * from the source map to the range dst_addr/len 1702 * in the destination map. 1703 * 1704 * This routine is only advisory and need not do anything. 1705 */ 1706 /* call deleted in <machine/pmap.h> */ 1707 1708 /* 1709 * pmap_update: [ INTERFACE ] 1710 * 1711 * Require that all active physical maps contain no 1712 * incorrect entries NOW, by processing any deferred 1713 * pmap operations. 1714 */ 1715 /* call deleted in <machine/pmap.h> */ 1716 1717 /* 1718 * pmap_activate: [ INTERFACE ] 1719 * 1720 * Activate the pmap used by the specified process. This includes 1721 * reloading the MMU context of the current process, and marking 1722 * the pmap in use by the processor. 1723 */ 1724 void 1725 pmap_activate(struct lwp *l) 1726 { 1727 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1728 1729 KASSERT(l == curlwp); 1730 1731 /* 1732 * If the pmap doesn't have a valid context, just use 1733 * the kernel's context for now (don't worry, the kernel's 1734 * mappings are protected with PME_K). 1735 */ 1736 PMAP_CRIT_ENTER(pmap->pm_busy++); 1737 (void) pmap_swap_context(pmap->pm_context != NULL 1738 ? pmap->pm_context->ctx_num : 0); 1739 if (pmap != pmap_kernel()) { 1740 TAILQ_REMOVE(&pmap_all_user_pmaps, pmap, pm_list); 1741 TAILQ_INSERT_TAIL(&pmap_all_user_pmaps, pmap, pm_list); 1742 } 1743 PMAP_CRIT_EXIT(); 1744 } 1745 1746 /* 1747 * pmap_deactivate: [ INTERFACE ] 1748 * 1749 * Mark that the pmap used by the specified process is no longer 1750 * in use by the processor. 1751 */ 1752 void 1753 pmap_deactivate(struct lwp *l) 1754 { 1755 pmap_t pmap = l->l_proc->p_vmspace->vm_map.pmap; 1756 1757 PMAP_CRIT_ENTER(); 1758 KASSERT(pmap->pm_busy != 0); 1759 PMAP_CRIT_EXIT(pmap->pm_busy--); 1760 } 1761 1762 static vaddr_t pmap_tmpmap_srcva; 1763 static vaddr_t pmap_tmpmap_dstva; 1764 1765 static unsigned int pmap_tmpmap_srcidx; 1766 static unsigned int pmap_tmpmap_dstidx; 1767 1768 /* 1769 * pmap_zero_page: [ INTERFACE ] 1770 * 1771 * Zero the specified VM page by mapping the page into the kernel 1772 * and using memset() (or equivalent) to clear its contents. 1773 */ 1774 void 1775 pmap_zero_page(paddr_t pa) 1776 { 1777 const int flags = PMAP_WIRED; 1778 1779 /* Build the new PME. */ 1780 const pm_entry_t dst_pme = 1781 pmap_make_pme(pmap_kernel(), pa, 1782 UVM_PROT_READ | UVM_PROT_WRITE, flags); 1783 1784 /* Set the new PME. */ 1785 KASSERT(! pme_valid_p(pgmmu_getpme(pmap_tmpmap_dstidx))); 1786 pgmmu_setpme(pmap_tmpmap_dstidx, dst_pme); 1787 1788 /* Zero the page. */ 1789 zeropage((void *)pmap_tmpmap_dstva); 1790 1791 /* Invalidate the PME. */ 1792 pgmmu_setpme(pmap_tmpmap_dstidx, 0); 1793 } 1794 1795 /* 1796 * pmap_copy_page: [ INTERFACE ] 1797 * 1798 * Copy the specified VM page by mapping the page(s) into the kernel 1799 * and using memcpy() (or equivalent). 1800 */ 1801 void 1802 pmap_copy_page(paddr_t src, paddr_t dst) 1803 { 1804 const int flags = PMAP_WIRED; 1805 1806 /* Build the new PMEs. */ 1807 const pm_entry_t src_pme = 1808 pmap_make_pme(pmap_kernel(), src, 1809 UVM_PROT_READ, flags); 1810 const pm_entry_t dst_pme = 1811 pmap_make_pme(pmap_kernel(), dst, 1812 UVM_PROT_READ | UVM_PROT_WRITE, flags); 1813 1814 /* Set the new PMEs. */ 1815 KASSERT(! pme_valid_p(pgmmu_getpme(pmap_tmpmap_srcidx))); 1816 pgmmu_setpme(pmap_tmpmap_srcidx, src_pme); 1817 KASSERT(! pme_valid_p(pgmmu_getpme(pmap_tmpmap_dstidx))); 1818 pgmmu_setpme(pmap_tmpmap_dstidx, dst_pme); 1819 1820 /* Copy the page. */ 1821 copypage((void *)pmap_tmpmap_srcva, (void *)pmap_tmpmap_dstva); 1822 1823 /* Invalidate the PMEs. */ 1824 pgmmu_setpme(pmap_tmpmap_srcidx, 0); 1825 pgmmu_setpme(pmap_tmpmap_dstidx, 0); 1826 } 1827 1828 /* 1829 * pmap_testbit: 1830 * 1831 * Test the modified / referenced bits of a physical page. 1832 */ 1833 static bool 1834 pmap_testbit(struct vm_page *pg, pm_entry_t bit) 1835 { 1836 struct pv_entry *pv; 1837 pm_entry_t pme; 1838 1839 PMAP_CRIT_ENTER(); 1840 1841 pme = VM_MDPAGE_MR(pg); 1842 1843 for (pv = VM_MDPAGE_PVS(pg); 1844 (pme & bit) == 0 && pv != NULL; pv = pv->pv_next) { 1845 pme |= pgmmu_getpme(pme_index(pv->pv_pmeg, PV_VA(pv))); 1846 } 1847 1848 VM_MDPAGE_ADD_MR(pg, pme); 1849 1850 PMAP_CRIT_EXIT(); 1851 1852 return (pme & bit) != 0; 1853 } 1854 1855 /* 1856 * pmap_is_referenced: [ INTERFACE ] 1857 * 1858 * Return whether or not the specified physical page has been referenced 1859 * by any physical maps. 1860 */ 1861 bool 1862 pmap_is_referenced(struct vm_page *pg) 1863 { 1864 return pmap_testbit(pg, PME_R); 1865 } 1866 1867 /* 1868 * pmap_is_modified: [ INTERFACE ] 1869 * 1870 * Return whether or not the specified physical page has been modified 1871 * by any physical maps. 1872 */ 1873 bool 1874 pmap_is_modified(struct vm_page *pg) 1875 { 1876 return pmap_testbit(pg, PME_M); 1877 } 1878 1879 /* 1880 * pmap_changebit: 1881 * 1882 * Test-and-change various bits (including mod/ref bits). 1883 * Returns the accumulated previously-set PME bits. 1884 */ 1885 static pm_entry_t 1886 pmap_changebit(struct vm_page *pg, pm_entry_t set, pm_entry_t mask) 1887 { 1888 struct pv_entry *pv; 1889 pm_entry_t combined_pme, opme, npme; 1890 unsigned int pmeidx; 1891 1892 PMAP_CRIT_ENTER(); 1893 1894 /* 1895 * Since we need to report if the page was mod/ref'd before 1896 * we cleared the bit, we need to seed ourself with the current 1897 * state in the vm_page in the event there are no mappings 1898 * left to enumerate. 1899 */ 1900 combined_pme = VM_MDPAGE_MR(pg); 1901 1902 /* 1903 * Since we're running over every mapping for the page anyway, 1904 * we might as well synchronize any attribute bits that we're 1905 * not clearing. 1906 */ 1907 for (pv = VM_MDPAGE_PVS(pg); pv != NULL; pv = pv->pv_next) { 1908 pmeidx = pme_index(pv->pv_pmeg, PV_VA(pv)); 1909 opme = pgmmu_getpme(pmeidx); 1910 npme = (opme | set) & mask; 1911 combined_pme |= opme; 1912 if (opme != npme) { 1913 pgmmu_setpme(pmeidx, npme); 1914 } 1915 } 1916 1917 /* 1918 * Update any attributes we looked at, clear the ones we're clearing. 1919 */ 1920 VM_MDPAGE_SET_MR(pg, (combined_pme | set) & mask); 1921 1922 PMAP_CRIT_EXIT(); 1923 1924 return combined_pme; 1925 } 1926 1927 /* 1928 * pmap_clear_modify: [ INTERFACE ] 1929 * 1930 * Clear the modify bits on the specified physical page. 1931 */ 1932 bool 1933 pmap_clear_modify(struct vm_page *pg) 1934 { 1935 return (pmap_changebit(pg, 0, (pm_entry_t)~PME_M) & PME_M) != 0; 1936 } 1937 1938 /* 1939 * pmap_clear_reference: [ INTERFACE ] 1940 * 1941 * Clear the reference bit on the specified physical page. 1942 */ 1943 bool 1944 pmap_clear_reference(struct vm_page *pg) 1945 { 1946 return (pmap_changebit(pg, 0, (pm_entry_t)~PME_R) & PME_R) != 0; 1947 } 1948 1949 /* 1950 * pmap_phys_address: [ INTERFACE ] 1951 * 1952 * Return the physical address corresponding to the specified 1953 * cookie. Used by the device pager to decode a device driver's 1954 * mmap entry point return value. 1955 */ 1956 paddr_t 1957 pmap_phys_address(paddr_t cookie) 1958 { 1959 return pgmmu_ptob(cookie); 1960 } 1961 1962 /* 1963 * pmap_init_kcore_hdr: 1964 * 1965 * Initialize the m68k kernel crash dump header with information 1966 * necessary to perform KVA -> phys translations. 1967 * 1968 * Returns a pointer to the crash dump RAM segment entries for 1969 * machine-specific code to initialize. 1970 */ 1971 phys_ram_seg_t * 1972 pmap_init_kcore_hdr(cpu_kcore_hdr_t *h) 1973 { 1974 return NULL; 1975 } 1976 1977 #if defined(DDB) || defined(KGDB) 1978 /* 1979 * pmap_db_write_text_enter: 1980 * 1981 * Temporarily map a page of kernel text read-write for the 1982 * kernel debugger. 1983 */ 1984 bool 1985 pmap_db_write_text_enter(vaddr_t pgva, struct pmap_db_write_text_context *ctx) 1986 { 1987 sm_entry_t sme = pgmmu_getsme0(pgva); 1988 if (! sme_valid_p(sme)) { 1989 return false; 1990 } 1991 1992 unsigned int pmeidx = pme_index(sme_pmeg(sme), pgva); 1993 pm_entry_t opme = pgmmu_getpme(pmeidx); 1994 if (! pme_valid_p(opme)) { 1995 return false; 1996 } 1997 1998 pm_entry_t npme = opme | PME_W; 1999 pgmmu_setpme(pmeidx, npme); 2000 2001 ctx->pmeidx = pmeidx; 2002 ctx->opme = opme; 2003 2004 return true; 2005 } 2006 2007 /* 2008 * pmap_db_write_text_exit: 2009 * 2010 * Undo the effects of pmap_db_write_text_enter(). 2011 */ 2012 void 2013 pmap_db_write_text_exit(struct pmap_db_write_text_context *ctx) 2014 { 2015 pgmmu_setpme(ctx->pmeidx, ctx->opme); 2016 } 2017 #endif /* DDB || KGDB */ 2018 2019 /***************************** PMAP BOOTSTRAP ********************************/ 2020 2021 extern char * kernel_text; 2022 extern char * etext; 2023 2024 static vaddr_t lwp0uarea; 2025 char * vmmap; 2026 2027 /* XXX Doesn't belong here. */ 2028 paddr_t avail_start; /* PA of first available physical page */ 2029 paddr_t avail_end; /* PA of last available physical page */ 2030 2031 /* 2032 * This structure is used to save firmware mappings that the kernel 2033 * can also use. 2034 */ 2035 struct pmap_static_mapping { 2036 vaddr_t psm_va; 2037 paddr_t psm_pa; 2038 size_t psm_size; 2039 pm_entry_t psm_pme; 2040 }; 2041 2042 #define MAX_STATIC_MAPPINGS 8 2043 static struct pmap_static_mapping static_mappings[MAX_STATIC_MAPPINGS]; 2044 static int num_static_mappings; 2045 2046 /* 2047 * pmap_add_static_mapping: 2048 * 2049 * Add a VA != PA static mapping to the table. This is done 2050 * page-by-page, and may extend an existing entry. 2051 */ 2052 static bool 2053 pmap_add_static_mapping(vaddr_t va, paddr_t pa, pm_entry_t pme) 2054 { 2055 int i; 2056 2057 /* only care about writability */ 2058 pme &= PME_W; 2059 2060 /* 2061 * First check to see if this extends an existing entry. 2062 */ 2063 for (i = 0; i < num_static_mappings; i++) { 2064 if (va == static_mappings[i].psm_va + PAGE_SIZE && 2065 pa == static_mappings[i].psm_pa + PAGE_SIZE && 2066 pme == static_mappings[i].psm_pme) { 2067 static_mappings[i].psm_size += PAGE_SIZE; 2068 return true; 2069 } 2070 } 2071 2072 /* 2073 * Create a new entry. 2074 */ 2075 if (num_static_mappings == MAX_STATIC_MAPPINGS) { 2076 return false; 2077 } 2078 2079 static_mappings[i].psm_va = va; 2080 static_mappings[i].psm_pa = pa; 2081 static_mappings[i].psm_pme = pme; 2082 static_mappings[i].psm_size = PAGE_SIZE; 2083 num_static_mappings++; 2084 2085 return true; 2086 } 2087 2088 /* 2089 * pmap_pa_has_static_mapping: 2090 * 2091 * Returns true if the specified PA (and length) has a static mapping 2092 * with the requested permission. PMAP_* flags corresponding to the 2093 * mapping's properties are returned via *flagsp. 2094 */ 2095 bool 2096 pmap_pa_has_static_mapping(paddr_t pa, size_t len, vm_prot_t prot, 2097 vaddr_t *vap, int *flagsp) 2098 { 2099 paddr_t lastpg = pgmmu_btop(pa + (len - 1)); 2100 paddr_t firstpg = pgmmu_btop(pa); 2101 paddr_t tfirst, tlast; 2102 bool need_write = !!(prot & UVM_PROT_WRITE); 2103 int i; 2104 2105 for (i = 0; i < num_static_mappings; i++) { 2106 tfirst = pgmmu_btop(static_mappings[i].psm_pa); 2107 tlast = pgmmu_btop(static_mappings[i].psm_pa + 2108 (static_mappings[i].psm_size - 1)); 2109 2110 if (firstpg >= tfirst && lastpg <= tlast) { 2111 if (need_write && 2112 (static_mappings[i].psm_pme & PME_W) == 0) { 2113 return false; 2114 } 2115 *vap = static_mappings[i].psm_va + 2116 (pa - static_mappings[i].psm_pa); 2117 /* 2118 * No systems with this MMU have a data cache, 2119 * so always indocate PMAP_NOCACHE to anyone 2120 * making inquiries. 2121 */ 2122 *flagsp = PMAP_NOCACHE; 2123 return true; 2124 } 2125 } 2126 2127 return false; 2128 } 2129 2130 /* 2131 * pmap_va_is_static_mapping: 2132 * 2133 * Returns true if the specified VA (and length) is a static 2134 * mapping. 2135 */ 2136 bool 2137 pmap_va_is_static_mapping(vaddr_t va, size_t len) 2138 { 2139 vaddr_t lastpg = pgmmu_btop(va + (len - 1)); 2140 vaddr_t firstpg = pgmmu_btop(va); 2141 vaddr_t tfirst, tlast; 2142 int i; 2143 2144 for (i = 0; i < num_static_mappings; i++) { 2145 tfirst = pgmmu_btop(static_mappings[i].psm_va); 2146 tlast = pgmmu_btop(static_mappings[i].psm_va + 2147 (static_mappings[i].psm_size - 1)); 2148 2149 if (firstpg >= tfirst && lastpg <= tlast) { 2150 return true; 2151 } 2152 } 2153 2154 return false; 2155 } 2156 2157 /* 2158 * pmap_bootstrap1: 2159 * 2160 * Phase 1 of bootstrapping virtual memory. For this implementation, 2161 * the MMU is already enabled and we are running on the mappings 2162 * set up by the firmware. We need to initialize some of our 2163 * data structures, and preserve / adjust some of the mappings that 2164 * already exist. 2165 * 2166 * N.B. reloff is unused in this implementation because the MMU 2167 * is already on and thus manual relocations are not necessary. 2168 */ 2169 paddr_t __attribute__((no_instrument_function)) 2170 pmap_bootstrap1(paddr_t nextpa, paddr_t reloff __unused) 2171 { 2172 int i, seg, pmeg; 2173 paddr_t pa; 2174 paddr_t lwp0upa; 2175 sm_entry_t sme; 2176 pm_entry_t pme; 2177 vaddr_t va; 2178 vaddr_t nextva; 2179 vaddr_t endva; 2180 vaddr_t alloc_startva; 2181 int entry_count = 0; 2182 2183 /* Initialize the kernel pmap. */ 2184 pmap_pinit(pmap_kernel(), &static_contexts[0]); 2185 pmap_kernel()->pm_busy = 1; /* kernel pmap starts out busy */ 2186 2187 /* Initialize the pmeg bitmap. */ 2188 pmap_bitmap_init(&pmeg_bitmap); 2189 2190 TAILQ_INIT(&pmap_all_user_pmaps); 2191 2192 /* 2193 * Time to tidy up all of the various mappings left for us by the 2194 * firmware. 2195 * 2196 * First pass through SegMap0, claim all of the pmegs that are 2197 * currently in-use; we'll use them for kernel mappings, and 2198 * they're all pre-allocated. 2199 */ 2200 for (seg = 0, va = 0; seg < PGMMU_NUM_SEGS; 2201 seg++, va += PGMMU_SEG_SIZE) { 2202 sme = pgmmu_getsme0(va); 2203 if (sme_valid_p(sme)) { 2204 pmap_bitmap_claim(&pmeg_bitmap, sme_pmeg(sme)); 2205 } 2206 } 2207 2208 /* 2209 * Now we know which pmegs are in-use, we can go through the 2210 * entire PageMap and zero-initialize all not-in-use entries. 2211 */ 2212 for (pmeg = 0; pmeg < PGMMU_NUM_PMEGS; pmeg++) { 2213 if (pmap_bitmap_claimed_p(&pmeg_bitmap, pmeg)) { 2214 continue; 2215 } 2216 for (i = 0; i < PGMMU_PMEG_SIZE; i++) { 2217 pgmmu_setpme((pmeg << PGMMU_PMEG_SHIFT) + i, 0); 2218 } 2219 } 2220 2221 /* 2222 * Now go back through SegMap0 and assign pmegs to each segment 2223 * that doesn't already have one. 2224 */ 2225 for (seg = 0, va = 0; seg < PGMMU_NUM_SEGS; 2226 seg++, va += PGMMU_SEG_SIZE) { 2227 sme = pgmmu_getsme0(va); 2228 if (! sme_valid_p(sme)) { 2229 pmeg = pmap_bitmap_alloc(&pmeg_bitmap); 2230 pgmmu_setsme0(va, SME_V | pmeg); 2231 } 2232 } 2233 2234 /* 2235 * Now go through the SegMaps for all of the user contexts 2236 * and zero-initialize them. 2237 */ 2238 for (i = 1; i < PGMMU_NUM_CONTEXTS; i++) { 2239 pgmmu_setcontext(i); 2240 for (seg = 0, va = 0; seg < PGMMU_NUM_SEGS; 2241 seg++, va += PGMMU_SEG_SIZE) { 2242 pgmmu_setsme(va, 0); 2243 } 2244 } 2245 2246 pgmmu_setcontext(0); 2247 2248 /* 2249 * The system firmware has done a few things: 2250 * 2251 * ==> (1) Mapped all base RAM VA==PA. The kernel has been loaded 2252 * here. We need to invalidate the mappings before and after 2253 * the kernel so that we can use the VA space for our own purposes. 2254 * NOTE: The stack that we were using when we entered the kernel 2255 * is located somewhere in there, so we need to have switched 2256 * to a temporary stack within the base kernel image before 2257 * getting to pmap_bootstrap1(). 2258 * 2259 * ==> (2) Mapped the ROM somewhere in the top 1MB of Context 0 2260 * (not necessarily the entire 1MB). 2261 * 2262 * ==> (3) Mapped the on-board devices and firmware reserved memory 2263 * just below the ROM. 2264 * 2265 * For (2) and (3), the mappings are not VA==PA. So, what we're 2266 * going to do is preserve all VA!=PA mappings, and then re-use them 2267 * whenever asked for them by others (like when drivers map devices), 2268 * and we'll clear the mappings for areas not-the-kernel in the 2269 * VA==PA areas. For the kernel text, we'll fix up the mappings to 2270 * be read-only. 2271 * 2272 * The physical pages before the kernel will be re-used for the 2273 * lwp0 u-area. We've arranged for the kernel to be linked at 2274 * 0 + USPACE in order to faciliate this, and we know this will 2275 * be in the first segment. 2276 * 2277 * For all mappings that we preserve, we also ensure that the K 2278 * bit is set. 2279 * 2280 * XXX We're making assumptions about the system firmware and memory 2281 * map here, but I can count on one finger the number of systems that 2282 * use this MMU. 2283 */ 2284 2285 /* Unmap the region before the kernel text. */ 2286 endva = pgmmu_trunc_page(&kernel_text); 2287 lwp0upa = endva - USPACE; 2288 for (va = 0; va < endva; va += PAGE_SIZE) { 2289 pmap_setkpme(va, 0); 2290 } 2291 2292 /* Fix kernel text to be read-only. */ 2293 endva = pgmmu_trunc_page(&etext); 2294 for (; va < endva; va += PAGE_SIZE) { 2295 pme = pmap_getkpme(va); 2296 pmap_setkpme(va, (pme & ~PME_W) | PME_K); 2297 entry_count++; 2298 } 2299 2300 /* Fixup priv on the rest of the kernel. */ 2301 endva = alloc_startva = nextpa = pgmmu_round_page(nextpa); 2302 for (; va < endva; va += PAGE_SIZE) { 2303 pme = pmap_getkpme(va); 2304 pmap_setkpme(va, pme | PME_W | PME_K); 2305 entry_count++; 2306 } 2307 2308 /* 2309 * Now walk every remaining kernel PME and check to see if 2310 * it's a VA != PA mapping. If so, preserve it (and clamp 2311 * the max kernel virtual address as necessary). 2312 */ 2313 for (; va < KERNEL_MAX_ADDRESS && va >= alloc_startva; 2314 va += PAGE_SIZE) { 2315 pme = pmap_getkpme(va); 2316 if (! pme_valid_p(pme)) { 2317 continue; 2318 } 2319 pa = pme_pa(pme); 2320 if (va == pa) { 2321 pmap_setkpme(va, 0); 2322 continue; 2323 } 2324 2325 /* Clamp the max kernel virtual address. */ 2326 if (va < kernel_virtual_max) { 2327 kernel_virtual_max = va; 2328 } 2329 2330 if (! pmap_add_static_mapping(va, pa, pme)) { 2331 /* XXX log a warning? */ 2332 } 2333 2334 /* Ensure it's kernel-only. */ 2335 pmap_setkpme(va, pme | PME_K); 2336 entry_count++; 2337 } 2338 2339 /* 2340 * Allocate / map some special purpose VAs: 2341 */ 2342 nextva = alloc_startva; 2343 2344 /* lwp0 u-area. */ 2345 lwp0uarea = nextva; 2346 nextva += USPACE; 2347 2348 pme = pmap_make_pme(pmap_kernel(), lwp0upa, 2349 UVM_PROT_READ|UVM_PROT_WRITE, PMAP_WIRED); 2350 for (va = lwp0uarea; va < nextva; va += PAGE_SIZE) { 2351 pmap_setkpme(va, pme); 2352 pme++; /* increment PFN field */ 2353 entry_count++; 2354 } 2355 2356 /* pmap temporary map addresses */ 2357 pmap_tmpmap_srcva = nextva; 2358 nextva += PAGE_SIZE; 2359 sme = pgmmu_getsme0(pmap_tmpmap_srcva); 2360 pmap_tmpmap_srcidx = pme_index(sme_pmeg(sme), pmap_tmpmap_srcva); 2361 2362 pmap_tmpmap_dstva = nextva; 2363 nextva += PAGE_SIZE; 2364 sme = pgmmu_getsme0(pmap_tmpmap_dstva); 2365 pmap_tmpmap_dstidx = pme_index(sme_pmeg(sme), pmap_tmpmap_dstva); 2366 2367 /* vmmap temporary map address */ 2368 vmmap = (char *)nextva; 2369 nextva += PAGE_SIZE; 2370 2371 /* kernel message buffer */ 2372 msgbufaddr = (char *)nextva; 2373 nextva += pgmmu_round_page(MSGBUFSIZE); 2374 2375 /* UVM-managed kernel virtual starts here. */ 2376 kernel_virtual_start = nextva; 2377 2378 /* 2379 * Record the number of wired mappings we create above 2380 * in the kernel pmap stats. 2381 */ 2382 pmap_kernel()->pm_stats.resident_count = entry_count; 2383 pmap_kernel()->pm_stats.wired_count = entry_count; 2384 2385 return nextpa; 2386 } 2387 2388 /* 2389 * pmap_bootstrap2: 2390 * 2391 * Phase 2 of bootstrapping virtual memory. For this implementation, 2392 * we just have to finish setting up some run-time-computed global 2393 * pmap data, plus the lwp0 u-area, curlwp, and curpcb. 2394 * 2395 * Returns the new kernel %sp value for lwp0. 2396 */ 2397 void * 2398 pmap_bootstrap2(void) 2399 { 2400 /* Early low-level UVM initialization. */ 2401 uvmexp.pagesize = PAGE_SIZE; 2402 uvm_md_init(); 2403 2404 /* Initialize lwp0 u-area, curlwp, and curpcb. */ 2405 memset((void *)lwp0uarea, 0, USPACE); 2406 uvm_lwp_setuarea(&lwp0, lwp0uarea); 2407 curlwp = &lwp0; 2408 curpcb = lwp_getpcb(&lwp0); 2409 2410 /* Create a fake exception frame so that cpu_lwp_fork() can copy it. */ 2411 struct trapframe *tf = (struct trapframe *)(lwp0uarea + USPACE) - 1; 2412 tf->tf_sr = PSL_USER; 2413 lwp0.l_md.md_regs = (int *)tf; 2414 2415 /* 2416 * Initialize the source/destination control registers for 2417 * movs. 2418 */ 2419 setsfc(FC_USERD); 2420 setdfc(FC_USERD); 2421 2422 return tf; 2423 } 2424
Indexes created Mon Aug 10 00:24:53 UTC 2026