1 /* $NetBSD: pmap_segtab.c,v 1.38 2026/08/23 22:08:41 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 1998, 2001 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center and by Chris G. Demetriou. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * Copyright (c) 1992, 1993 35 * The Regents of the University of California. All rights reserved. 36 * 37 * This code is derived from software contributed to Berkeley by 38 * the Systems Programming Group of the University of Utah Computer 39 * Science Department and Ralph Campbell. 40 * 41 * Redistribution and use in source and binary forms, with or without 42 * modification, are permitted provided that the following conditions 43 * are met: 44 * 1. Redistributions of source code must retain the above copyright 45 * notice, this list of conditions and the following disclaimer. 46 * 2. Redistributions in binary form must reproduce the above copyright 47 * notice, this list of conditions and the following disclaimer in the 48 * documentation and/or other materials provided with the distribution. 49 * 3. Neither the name of the University nor the names of its contributors 50 * may be used to endorse or promote products derived from this software 51 * without specific prior written permission. 52 * 53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 63 * SUCH DAMAGE. 64 * 65 * @(#)pmap.c 8.4 (Berkeley) 1/26/94 66 */ 67 68 #include <sys/cdefs.h> 69 70 __KERNEL_RCSID(0, "$NetBSD: pmap_segtab.c,v 1.38 2026/08/23 22:08:41 riastradh Exp $"); 71 72 /* 73 * Manages physical address maps. 74 * 75 * In addition to hardware address maps, this 76 * module is called upon to provide software-use-only 77 * maps which may or may not be stored in the same 78 * form as hardware maps. These pseudo-maps are 79 * used to store intermediate results from copy 80 * operations to and from address spaces. 81 * 82 * Since the information managed by this module is 83 * also stored by the logical address mapping module, 84 * this module may throw away valid virtual-to-physical 85 * mappings at almost any time. However, invalidations 86 * of virtual-to-physical mappings must be done as 87 * requested. 88 * 89 * In order to cope with hardware architectures which 90 * make virtual-to-physical map invalidates expensive, 91 * this module may delay invalidate or reduced protection 92 * operations until such time as they are actually 93 * necessary. This module is given full information as 94 * to which processors are currently using which maps, 95 * and to when physical maps must be made correct. 96 */ 97 98 #define __PMAP_PRIVATE 99 100 #include "opt_multiprocessor.h" 101 102 #include <sys/param.h> 103 104 #include <sys/atomic.h> 105 #include <sys/mutex.h> 106 #include <sys/proc.h> 107 #include <sys/systm.h> 108 109 #include <uvm/uvm.h> 110 #include <uvm/pmap/pmap.h> 111 112 #if defined(XSEGSHIFT) && XSEGSHIFT == SEGSHIFT 113 #undef XSEGSHIFT 114 #undef XSEGLENGTH 115 #undef NBXSEG 116 #undef NXSEGPG 117 #endif 118 119 #define MULT_CTASSERT(a,b) __CTASSERT((a) < (b) || ((a) % (b) == 0)) 120 121 __CTASSERT(sizeof(pmap_ptpage_t) == NBPG); 122 123 #if defined(PMAP_HWPAGEWALKER) 124 #ifdef _LP64 125 MULT_CTASSERT(PMAP_PDETABSIZE, NPDEPG); 126 MULT_CTASSERT(NPDEPG, PMAP_PDETABSIZE); 127 #endif /* _LP64 */ 128 MULT_CTASSERT(sizeof(pmap_pdetab_t *), sizeof(pd_entry_t)); 129 MULT_CTASSERT(sizeof(pd_entry_t), sizeof(pmap_pdetab_t)); 130 131 #if 0 132 #ifdef _LP64 133 static const bool separate_pdetab_root_p = NPDEPG != PMAP_PDETABSIZE; 134 #else 135 static const bool separate_pdetab_root_p = true; 136 #endif /* _LP64 */ 137 #endif 138 139 typedef struct { 140 pmap_pdetab_t *free_pdetab0; /* free list kept locally */ 141 pmap_pdetab_t *free_pdetab; /* free list kept locally */ 142 #ifdef DEBUG 143 uint32_t nget; 144 uint32_t nput; 145 uint32_t npage; 146 #define PDETAB_ADD(n, v) (pmap_segtab_info.pdealloc.n += (v)) 147 #else 148 #define PDETAB_ADD(n, v) ((void) 0) 149 #endif /* DEBUG */ 150 } pmap_pdetab_alloc_t; 151 #endif /* PMAP_HWPAGEWALKER */ 152 153 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 154 #ifdef _LP64 155 __CTASSERT(NSEGPG >= PMAP_SEGTABSIZE); 156 __CTASSERT(NSEGPG % PMAP_SEGTABSIZE == 0); 157 #endif 158 __CTASSERT(NBPG >= sizeof(pmap_segtab_t)); 159 160 typedef struct { 161 pmap_segtab_t *free_segtab0; /* free list kept locally */ 162 pmap_segtab_t *free_segtab; /* free list kept locally */ 163 #ifdef DEBUG 164 uint32_t nget; 165 uint32_t nput; 166 uint32_t npage; 167 #define SEGTAB_ADD(n, v) (pmap_segtab_info.segalloc.n += (v)) 168 #else 169 #define SEGTAB_ADD(n, v) ((void) 0) 170 #endif 171 } pmap_segtab_alloc_t; 172 #endif /* !PMAP_HWPAGEWALKER || !PMAP_MAP_PDETABPAGE */ 173 174 struct pmap_segtab_info { 175 #if defined(PMAP_HWPAGEWALKER) 176 pmap_pdetab_alloc_t pdealloc; 177 #endif 178 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 179 pmap_segtab_alloc_t segalloc; 180 #endif 181 #ifdef PMAP_PPG_CACHE 182 struct pgflist ptp_pgflist; /* Keep a list of idle page tables. */ 183 #endif 184 } pmap_segtab_info = { 185 #ifdef PMAP_PPG_CACHE 186 .ptp_pgflist = LIST_HEAD_INITIALIZER(pmap_segtab_info.ptp_pgflist), 187 #endif 188 }; 189 190 kmutex_t pmap_segtab_lock __cacheline_aligned; 191 192 #ifndef PMAP_HWPAGEWALKER 193 /* 194 * Check that a seg_ppg[] array is empty. 195 * 196 * This is used when allocating or freeing a pmap_segtab_t. The stb 197 * should be unused -- meaning, none of the seg_ppg[] pointers are 198 * not NULL, as it transitions from either freshly allocated segtab from 199 * pmap pool, an unused allocated page segtab alloc from the SMP case, 200 * where two CPUs attempt to allocate the same underlying segtab, the 201 * release of a segtab entry to the freelist, or for SMP, where reserve 202 * also frees a freshly allocated but unused entry. 203 */ 204 static void 205 pmap_check_stb(pmap_segtab_t *stb, const char *caller, const char *why) 206 { 207 #ifdef DEBUG 208 for (size_t i = 0; i < PMAP_SEGTABSIZE; i++) { 209 if (stb->seg_ppg[i] != NULL) { 210 #define DEBUG_NOISY 211 #ifdef DEBUG_NOISY 212 UVMHIST_FUNC(__func__); 213 UVMHIST_CALLARGS(pmapxtabhist, "stb=%#jx", 214 (uintptr_t)stb, 0, 0, 0); 215 for (size_t j = i; j < PMAP_SEGTABSIZE; j++) 216 if (stb->seg_ppg[j] != NULL) 217 printf("%s: stb->seg_ppg[%zu] = %p\n", 218 caller, j, stb->seg_ppg[j]); 219 #endif 220 panic("%s: pm_segtab.seg_ppg[%zu] != 0 (%p): %s", 221 caller, i, stb->seg_ppg[i], why); 222 } 223 } 224 #endif 225 } 226 #endif /* PMAP_HWPAGEWALKER */ 227 228 static inline struct vm_page * 229 pmap_pte_pagealloc(void) 230 { 231 struct vm_page *pg; 232 233 pg = pmap_md_alloc_poolpage(UVM_PGA_ZERO | UVM_PGA_USERESERVE); 234 if (pg) { 235 #ifdef UVM_PAGE_TRKOWN 236 pg->owner_tag = NULL; 237 #endif 238 UVM_PAGE_OWN(pg, "pmap-ptp"); 239 } 240 241 return pg; 242 } 243 244 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 245 static vaddr_t 246 pmap_pde_to_va(pd_entry_t pde) 247 { 248 if (!pte_pde_valid_p(pde)) 249 return 0; 250 251 paddr_t pa = pte_pde_to_paddr(pde); 252 return pmap_md_direct_map_paddr(pa); 253 } 254 255 #ifdef _LP64 256 static pmap_pdetab_t * 257 pmap_pde_to_pdetab(pd_entry_t pde) 258 { 259 260 return (pmap_pdetab_t *)pmap_pde_to_va(pde); 261 } 262 #endif 263 264 static pmap_ptpage_t * 265 pmap_pde_to_ptpage(pd_entry_t pde) 266 { 267 268 return (pmap_ptpage_t *)pmap_pde_to_va(pde); 269 } 270 #endif 271 272 #ifdef _LP64 273 __CTASSERT((XSEGSHIFT - SEGSHIFT) % (PGSHIFT-3) == 0); 274 #endif 275 276 static inline pmap_ptpage_t * 277 pmap_ptpage(struct pmap *pmap, vaddr_t va) 278 { 279 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 280 vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1; 281 pmap_pdetab_t *ptb = pmap->pm_pdetab; 282 283 // UVMHIST_LOG(pmapxtabhist, "pm_pdetab %#jx", ptb, 0, 0, 0); 284 285 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va), 286 "pmap_kernel: %s, va %#" PRIxVADDR, 287 pmap == pmap_kernel() ? "true" : "false", 288 pmap == pmap_kernel() ? va : 0); 289 290 #ifdef _LP64 291 for (size_t segshift = XSEGSHIFT; 292 segshift > SEGSHIFT; 293 segshift -= PGSHIFT - 3, pdetab_mask = NSEGPG - 1) { 294 ptb = pmap_pde_to_pdetab(ptb->pde_pde[(va >> segshift) & pdetab_mask]); 295 if (ptb == NULL) 296 return NULL; 297 } 298 #endif 299 return pmap_pde_to_ptpage(ptb->pde_pde[(va >> SEGSHIFT) & pdetab_mask]); 300 #else 301 vaddr_t segtab_mask = PMAP_SEGTABSIZE - 1; 302 pmap_segtab_t *stb = pmap->pm_segtab; 303 304 KASSERTMSG(pmap != pmap_kernel() || !pmap_md_direct_mapped_vaddr_p(va), 305 "pmap %p va %#" PRIxVADDR, pmap, va); 306 #ifdef _LP64 307 for (size_t segshift = XSEGSHIFT; 308 segshift > SEGSHIFT; 309 segshift -= PGSHIFT - 3, segtab_mask = NSEGPG - 1) { 310 stb = stb->seg_seg[(va >> segshift) & segtab_mask]; 311 if (stb == NULL) 312 return NULL; 313 } 314 #endif 315 return stb->seg_ppg[(va >> SEGSHIFT) & segtab_mask]; 316 #endif 317 } 318 319 #if defined(PMAP_HWPAGEWALKER) 320 bool 321 pmap_pdetab_fixup(struct pmap *pmap, vaddr_t va) 322 { 323 struct pmap * const kpm = pmap_kernel(); 324 pmap_pdetab_t * const kptb = kpm->pm_pdetab; 325 pmap_pdetab_t * const uptb = pmap->pm_pdetab; 326 size_t idx = PMAP_PDETABSIZE - 1; 327 #if !defined(PMAP_MAP_PDETABPAGE) 328 __CTASSERT(PMAP_PDETABSIZE == PMAP_SEGTABSIZE); 329 pmap_segtab_t * const kstb = &pmap_kern_segtab; 330 pmap_segtab_t * const ustb = pmap->pm_segtab; 331 #endif 332 333 // Regardless of how many levels deep this page table is, we only 334 // need to verify the first level PDEs match up. 335 #ifdef XSEGSHIFT 336 idx &= va >> XSEGSHIFT; 337 #else 338 idx &= va >> SEGSHIFT; 339 #endif 340 if (uptb->pde_pde[idx] != kptb->pde_pde[idx]) { 341 pte_pde_set(&uptb->pde_pde[idx], kptb->pde_pde[idx]); 342 #if !defined(PMAP_MAP_PDETABPAGE) 343 ustb->seg_seg[idx] = kstb->seg_seg[idx]; // copy KVA of PTP 344 #endif 345 return true; 346 } 347 return false; 348 } 349 #endif /* PMAP_HWPAGEWALKER */ 350 351 352 static void 353 pmap_page_attach(pmap_t pmap, vaddr_t kva, struct vm_page *pg, 354 struct pglist *pglist, voff_t off) 355 { 356 UVMHIST_FUNC(__func__); 357 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx kva %#jx pg %#jx list %#jx", 358 (uintptr_t)pmap, (uintptr_t)kva, (uintptr_t)pg, (uintptr_t)pglist); 359 360 struct uvm_object * const uobj = &pmap->pm_uobject; 361 if (pg == NULL) { 362 paddr_t pa; 363 364 bool ok __diagused = pmap_extract(pmap_kernel(), kva, &pa); 365 KASSERT(ok); 366 367 pg = PHYS_TO_VM_PAGE(pa); 368 KASSERT(pg != NULL); 369 } 370 371 UVMHIST_LOG(pmapxtabhist, "kva %#jx uobj %#jx pg %#jx list %#jx", 372 (uintptr_t)kva, (uintptr_t)uobj, (uintptr_t)pg, (uintptr_t)pglist); 373 374 pmap_lock(pmap); 375 TAILQ_INSERT_TAIL(pglist, pg, pageq.queue); 376 uobj->uo_npages++; 377 pmap_unlock(pmap); 378 379 /* 380 * Now set each vm_page that maps this page to point to the 381 * pmap and set the offset to what we want. 382 */ 383 KASSERTMSG(pg->uobject == NULL, "pg %p pg->uobject %p", pg, pg->uobject); 384 pg->uobject = uobj; 385 pg->offset = off; 386 } 387 388 static struct vm_page * 389 pmap_page_detach(pmap_t pmap, struct pglist *list, vaddr_t va) 390 { 391 UVMHIST_FUNC(__func__); 392 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx kva %#jx list %#jx", 393 (uintptr_t)pmap, (uintptr_t)va, (uintptr_t)list, 0); 394 395 paddr_t pa; 396 bool ok __diagused = pmap_extract(pmap_kernel(), va, &pa); 397 KASSERT(ok); 398 399 struct vm_page * const pg = PHYS_TO_VM_PAGE(pa); 400 struct uvm_object * const uobj = &pmap->pm_uobject; 401 402 UVMHIST_LOG(pmapxtabhist, "kva %#jx uobj %#jx pg %#jx list %#jx", 403 (uintptr_t)va, (uintptr_t)uobj, (uintptr_t)pg, (uintptr_t)list); 404 405 KASSERTMSG(pg->uobject == uobj, "pg->uobject %p vs uobj %p", 406 pg->uobject, uobj); 407 408 pmap_lock(pmap); 409 TAILQ_REMOVE(list, pg, pageq.queue); 410 uobj->uo_npages--; 411 pmap_unlock(pmap); 412 413 pg->uobject = NULL; 414 pg->offset = 0; 415 416 return pg; 417 } 418 419 #ifndef PMAP_PPG_CACHE 420 static void 421 pmap_segtab_pagefree(pmap_t pmap, struct pglist *list, vaddr_t kva, size_t size) 422 { 423 #ifdef PMAP_MAP_PTEPAGE 424 UVMHIST_FUNC(__func__); 425 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx list %#jx kva %#jx size %#jx", 426 (uintptr_t)pmap, (uintptr_t)list, kva, size); 427 KASSERT(size == PAGE_SIZE); 428 if (size == PAGE_SIZE) { 429 UVMHIST_LOG(pmapxtabhist, "about to detach (kva %#jx)", 430 kva, 0, 0, 0); 431 uvm_pagefree(pmap_page_detach(pmap, list, kva)); 432 return; 433 } 434 #endif 435 for (size_t i = 0; i < size; i += PAGE_SIZE) { 436 (void)pmap_page_detach(pmap, list, kva + i); 437 } 438 439 uvm_km_free(kernel_map, kva, size, UVM_KMF_WIRED); 440 } 441 #endif 442 443 pt_entry_t * 444 pmap_pte_lookup(pmap_t pmap, vaddr_t va) 445 { 446 pmap_ptpage_t * const ppg = pmap_ptpage(pmap, va); 447 if (ppg == NULL) 448 return NULL; 449 450 const size_t pte_idx = (va >> PGSHIFT) & (NPTEPG - 1); 451 452 return ppg->ppg_ptes + pte_idx; 453 } 454 455 456 static pmap_ptpage_t * 457 pmap_ptpage_alloc(pmap_t pmap, int flags, paddr_t *pa_p) 458 { 459 UVMHIST_FUNC(__func__); 460 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx flags %#jx pa_p %#jx", (uintptr_t)pmap, 461 (uintptr_t)flags, (uintptr_t)pa_p, 0); 462 463 pmap_ptpage_t *ppg = NULL; 464 465 #ifdef PMAP_MAP_PTEPAGE 466 struct vm_page *pg = NULL; 467 paddr_t pa; 468 #ifdef PMAP_PPG_CACHE 469 ppg = pmap_pgcache_alloc(&pmap_segtab_info.ppg_flist); 470 #endif 471 if (ppg == NULL) { 472 pg = pmap_pte_pagealloc(); 473 if (pg == NULL) { 474 if (flags & PMAP_CANFAIL) 475 return NULL; 476 panic("%s: cannot allocate page table page ", 477 __func__); 478 } 479 pa = VM_PAGE_TO_PHYS(pg); 480 ppg = (pmap_ptpage_t *)PMAP_MAP_PTEPAGE(pa); 481 } else { 482 bool ok __diagused = pmap_extract(pmap_kernel(), (vaddr_t)ppg, &pa); 483 KASSERT(ok); 484 } 485 486 UVMHIST_LOG(pmapxtabhist, "about to attach", 0, 0, 0, 0); 487 pmap_page_attach(pmap, (vaddr_t)ppg, pg, &pmap->pm_ppg_list, 0); 488 489 *pa_p = pa; 490 #else 491 vaddr_t kva = uvm_km_alloc(kernel_map, PAGE_SIZE, PAGE_SIZE, 492 UVM_KMF_WIRED | UVM_KMF_WAITVA 493 | (flags & PMAP_CANFAIL ? UVM_KMF_CANFAIL : 0)); 494 if (kva == 0) { 495 if (flags & PMAP_CANFAIL) 496 return NULL; 497 panic("%s: cannot allocate page table page", __func__); 498 } 499 UVMHIST_LOG(pmapxtabhist, "about to attach", 0, 0, 0, 0); 500 pmap_page_attach(pmap, kva, NULL, &pmap->pm_ppg_list, 0); 501 ppg = (pmap_ptpage_t *)kva; 502 #endif 503 504 UVMHIST_LOG(pmapxtabhist, "... ppg %#jx", (uintptr_t)ppg, 0, 0, 0); 505 506 return ppg; 507 } 508 509 static void 510 pmap_ptpage_free(pmap_t pmap, pmap_ptpage_t *ppg, const char *caller) 511 { 512 UVMHIST_FUNC(__func__); 513 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx va %#jx", (uintptr_t)pmap, 514 (uintptr_t)ppg, 0, 0); 515 516 const vaddr_t kva = (vaddr_t)ppg; 517 /* 518 * All pte arrays should be page aligned. 519 */ 520 if ((kva & PAGE_MASK) != 0) { 521 panic("%s: pte entry at %p not page aligned", caller, ppg); 522 } 523 524 #ifdef DEBUG 525 for (size_t j = 0; j < NPTEPG; j++) { 526 if (ppg->ppg_ptes[j] != 0) { 527 UVMHIST_LOG(pmapxtabhist, 528 "pte entry %#jx not 0 (%#jx)", 529 (uintptr_t)&ppg->ppg_ptes[j], 530 (uintptr_t)ppg->ppg_ptes[j], 0, 0); 531 for (size_t i = j + 1; i < NPTEPG; i++) 532 if (ppg->ppg_ptes[i] != 0) 533 UVMHIST_LOG(pmapxtabhist, 534 "pte[%zu] = %#"PRIxPTE, 535 i, ppg->ppg_ptes[i], 0, 0); 536 537 panic("%s: pte entry at %p not 0 (%#" PRIxPTE ")", 538 __func__, &ppg->ppg_ptes[j], 539 ppg->ppg_ptes[j]); 540 } 541 } 542 #endif 543 //pmap_md_vca_clean(pg, (vaddr_t)ppg, NBPG); 544 #ifdef PMAP_PPG_CACHE 545 UVMHIST_LOG(pmapxtabhist, "about to detach", 0, 0, 0, 0); 546 pmap_page_detach(pmap, &pmap->pm_ppg_list, kva); 547 pmap_segtab_pagecache(&pmap_segtab_info.ppg_flist, ppg); 548 #else 549 pmap_segtab_pagefree(pmap, &pmap->pm_ppg_list, kva, PAGE_SIZE); 550 #endif /* PMAP_PPG_CACHE */ 551 } 552 553 554 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 555 556 static pmap_pdetab_t * 557 pmap_pdetab_alloc(struct pmap *pmap) 558 { 559 UVMHIST_FUNC(__func__); 560 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0); 561 562 pmap_pdetab_t *ptb; 563 #ifdef UVMHIST 564 bool found_on_freelist = false; 565 #endif 566 567 again: 568 mutex_spin_enter(&pmap_segtab_lock); 569 UVMHIST_LOG(pmapxtabhist, "free_pdetab %#jx", 570 (uintptr_t)pmap_segtab_info.pdealloc.free_pdetab, 0, 0, 0); 571 if (__predict_true((ptb = pmap_segtab_info.pdealloc.free_pdetab) != NULL)) { 572 pmap_segtab_info.pdealloc.free_pdetab = ptb->pde_next; 573 574 UVMHIST_LOG(pmapxtabhist, "freelist ptb=%#jx", 575 (uintptr_t)ptb, 0, 0, 0); 576 577 PDETAB_ADD(nget, 1); 578 ptb->pde_next = NULL; 579 #ifdef UVMHIST 580 found_on_freelist = true; 581 #endif 582 } 583 mutex_spin_exit(&pmap_segtab_lock); 584 585 struct vm_page *ptb_pg = NULL; 586 if (__predict_false(ptb == NULL)) { 587 uint64_t ticket = uvm_wait_prepare(); 588 589 ptb_pg = pmap_pte_pagealloc(); 590 591 UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx", 592 (uintptr_t)ptb_pg, 0, 0, 0); 593 if (__predict_false(ptb_pg == NULL)) { 594 /* 595 * XXX What else can we do? Could we deadlock here? 596 */ 597 uvm_wait("pdetab", ticket); 598 goto again; 599 } 600 601 UVMHIST_LOG(pmapxtabhist, "ptb_pg=%#jx 2", 602 (uintptr_t)ptb_pg, 0, 0, 0); 603 PDETAB_ADD(npage, 1); 604 const paddr_t ptb_pa = VM_PAGE_TO_PHYS(ptb_pg); 605 UVMHIST_LOG(pmapxtabhist, "ptb_pa=%#jx", (uintptr_t)ptb_pa, 0, 0, 0); 606 ptb = (pmap_pdetab_t *)PMAP_MAP_PDETABPAGE(ptb_pa); 607 UVMHIST_LOG(pmapxtabhist, "new ptb=%#jx", (uintptr_t)ptb, 0, 608 0, 0); 609 610 if (pte_invalid_pde() != 0) { 611 for (size_t i = 0; i < NPDEPG; i++) { 612 ptb->pde_pde[i] = pte_invalid_pde(); 613 } 614 } 615 } 616 617 UVMHIST_LOG(pmapxtabhist, "about to attach", 0, 0, 0, 0); 618 pmap_page_attach(pmap, (vaddr_t)ptb, ptb_pg, &pmap->pm_pdetab_list, 0); 619 620 UVMHIST_LOG(pmapxtabhist, "... ptb %#jx found on freelist %d", 621 (uintptr_t)ptb, found_on_freelist, 0, 0); 622 623 return ptb; 624 } 625 626 627 #else 628 /* 629 * Create and return a physical map. 630 * 631 * If the size specified for the map 632 * is zero, the map is an actual physical 633 * map, and may be referenced by the 634 * hardware. 635 * 636 * If the size specified is non-zero, 637 * the map will be used in software only, and 638 * is bounded by that size. 639 */ 640 static pmap_segtab_t * 641 pmap_segtab_alloc(struct pmap *pmap) 642 { 643 UVMHIST_FUNC(__func__); 644 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0); 645 646 pmap_segtab_t *stb; 647 bool found_on_freelist = false; 648 649 again: 650 mutex_spin_enter(&pmap_segtab_lock); 651 if (__predict_true((stb = pmap_segtab_info.segalloc.free_segtab) != NULL)) { 652 pmap_segtab_info.segalloc.free_segtab = stb->seg_next; 653 SEGTAB_ADD(nget, 1); 654 stb->seg_next = NULL; 655 found_on_freelist = true; 656 UVMHIST_LOG(pmapxtabhist, "freelist stb=%#jx", 657 (uintptr_t)stb, 0, 0, 0); 658 } 659 mutex_spin_exit(&pmap_segtab_lock); 660 661 struct vm_page *stb_pg = NULL; 662 if (__predict_false(stb == NULL)) { 663 uint64_t ticket = uvm_wait_prepare(); 664 665 stb_pg = pmap_pte_pagealloc(); 666 667 if (__predict_false(stb_pg == NULL)) { 668 /* 669 * XXX What else can we do? Could we deadlock here? 670 */ 671 uvm_wait("segtab", ticket); 672 goto again; 673 } 674 SEGTAB_ADD(npage, 1); 675 const paddr_t stb_pa = VM_PAGE_TO_PHYS(stb_pg); 676 677 stb = (pmap_segtab_t *)PMAP_MAP_SEGTABPAGE(stb_pa); 678 UVMHIST_LOG(pmapxtabhist, "new stb=%#jx", (uintptr_t)stb, 0, 679 0, 0); 680 #if 0 681 CTASSERT(NBPG / sizeof(*stb) == 1); 682 const size_t n = NBPG / sizeof(*stb); 683 if (n > 1) { 684 /* 685 * link all the segtabs in this page together 686 */ 687 for (size_t i = 1; i < n - 1; i++) { 688 stb[i].seg_next = &stb[i + 1]; 689 } 690 /* 691 * Now link the new segtabs into the free segtab list. 692 */ 693 mutex_spin_enter(&pmap_segtab_lock); 694 stb[n - 1].seg_next = pmap_segtab_info.segalloc.free_segtab; 695 pmap_segtab_info.segalloc.free_segtab = stb + 1; 696 SEGTAB_ADD(nput, n - 1); 697 mutex_spin_exit(&pmap_segtab_lock); 698 } 699 #endif 700 } 701 702 UVMHIST_LOG(pmapxtabhist, "about to attach", 0, 0, 0, 0); 703 pmap_page_attach(pmap, (vaddr_t)stb, stb_pg, &pmap->pm_segtab_list, 0); 704 705 pmap_check_stb(stb, __func__, 706 found_on_freelist ? "from free list" : "allocated"); 707 708 UVMHIST_LOG(pmapxtabhist, "... stb %#jx found on freelist %zu", 709 (uintptr_t)stb, found_on_freelist, 0, 0); 710 711 return stb; 712 } 713 #endif 714 715 #if defined(PMAP_HWPAGEWALKER) 716 static void 717 pmap_pdetab_free(pmap_pdetab_t *ptb) 718 { 719 UVMHIST_FUNC(__func__); 720 UVMHIST_CALLARGS(pmapxtabhist, "ptb %#jx", (uintptr_t)ptb, 0, 0, 0); 721 /* 722 * Insert the pdetab into the pdetab freelist. 723 */ 724 mutex_spin_enter(&pmap_segtab_lock); 725 ptb->pde_next = pmap_segtab_info.pdealloc.free_pdetab; 726 pmap_segtab_info.pdealloc.free_pdetab = ptb; 727 PDETAB_ADD(nput, 1); 728 mutex_spin_exit(&pmap_segtab_lock); 729 730 } 731 #endif 732 733 734 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 735 /* 736 * Insert the segtab into the segtab freelist. 737 */ 738 static void 739 pmap_segtab_free(pmap_segtab_t *stb) 740 { 741 UVMHIST_FUNC(__func__); 742 UVMHIST_CALLARGS(pmapxtabhist, "stb %#jx", (uintptr_t)stb, 0, 0, 0); 743 744 /* 745 * Insert the segtab into the segtab freelist. 746 */ 747 mutex_spin_enter(&pmap_segtab_lock); 748 stb->seg_next = pmap_segtab_info.segalloc.free_segtab; 749 pmap_segtab_info.segalloc.free_segtab = stb; 750 SEGTAB_ADD(nput, 1); 751 mutex_spin_exit(&pmap_segtab_lock); 752 } 753 #endif 754 755 #if defined(PMAP_HWPAGEWALKER) 756 static void 757 pmap_pdetab_release(pmap_t pmap, pmap_pdetab_t **ptb_p, bool free_ptb, 758 vaddr_t va, vsize_t vinc) 759 { 760 const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1; 761 pmap_pdetab_t *ptb = *ptb_p; 762 763 UVMHIST_FUNC(__func__); 764 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx ptb_p %#jx ptb %#jx free %jd", 765 (uintptr_t)pmap, (uintptr_t)ptb_p, (uintptr_t)ptb, free_ptb); 766 UVMHIST_LOG(pmapxtabhist, " va=%#jx vinc=%#jx", 767 (uintptr_t)va, (uintptr_t)vinc, 0, 0); 768 769 for (size_t i = (va / vinc) & pdetab_mask; 770 i < PMAP_PDETABSIZE; 771 i++, va += vinc) { 772 #ifdef _LP64 773 if (vinc > NBSEG) { 774 if (pte_pde_valid_p(ptb->pde_pde[i])) { 775 pmap_pdetab_t *nptb = 776 pmap_pde_to_pdetab(ptb->pde_pde[i]); 777 UVMHIST_LOG(pmapxtabhist, 778 " va %#jx ptp->pde_pde[%jd] (*%#jx) = %#jx " 779 "recursing", va, i, &ptb->pde_pde[i], 780 ptb->pde_pde[i]); 781 pmap_pdetab_release(pmap, &nptb, true, 782 va, vinc / NPDEPG); 783 ptb->pde_pde[i] = pte_invalid_pde(); 784 KASSERT(nptb == NULL); 785 } 786 continue; 787 } 788 #endif 789 KASSERT(vinc == NBSEG); 790 791 /* get pointer to PT page */ 792 pmap_ptpage_t *ppg = pmap_pde_to_ptpage(ptb->pde_pde[i]); 793 UVMHIST_LOG(pmapxtabhist, 794 " va %#jx ptb->pde_pde[%jd] (*%#jx) = %#jx", va, i, 795 (uintptr_t)&ptb->pde_pde[i], ptb->pde_pde[i]); 796 if (ppg == NULL) 797 continue; 798 799 UVMHIST_LOG(pmapxtabhist, " zeroing tab (%#jx)[%jd] (%#jx)", 800 (uintptr_t)ptb->pde_pde, i, (uintptr_t)&ptb->pde_pde[i], 0); 801 802 ptb->pde_pde[i] = pte_invalid_pde(); 803 804 pmap_ptpage_free(pmap, ppg, __func__); 805 } 806 807 if (free_ptb) { 808 UVMHIST_LOG(pmapxtabhist, " ptbp %#jx ptb %#jx", 809 (uintptr_t)ptb_p, (uintptr_t)ptb, 0, 0); 810 const vaddr_t kva = (vaddr_t)ptb; 811 UVMHIST_LOG(pmapxtabhist, "about to detach", 0, 0, 0, 0); 812 pmap_page_detach(pmap, &pmap->pm_pdetab_list, kva); 813 pmap_pdetab_free(ptb); 814 *ptb_p = NULL; 815 } 816 } 817 #endif 818 819 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 820 static void 821 pmap_segtab_release(pmap_t pmap, pmap_segtab_t **stb_p, bool free_stb, 822 pte_callback_t callback, uintptr_t flags, vaddr_t va, vsize_t vinc) 823 { 824 pmap_segtab_t *stb = *stb_p; 825 826 UVMHIST_FUNC(__func__); 827 UVMHIST_CALLARGS(pmapxtabhist, "pm=%#jx stb_p=%#jx free=%jd", 828 (uintptr_t)pmap, (uintptr_t)stb, free_stb, 0); 829 UVMHIST_LOG(pmapxtabhist, " callback=%#jx flags=%#jx va=%#jx vinc=%#jx", 830 (uintptr_t)callback, flags, (uintptr_t)va, (uintptr_t)vinc); 831 832 for (size_t i = (va / vinc) & (PMAP_SEGTABSIZE - 1); 833 i < PMAP_SEGTABSIZE; 834 i++, va += vinc) { 835 #ifdef _LP64 836 if (vinc > NBSEG) { 837 if (stb->seg_seg[i] != NULL) { 838 UVMHIST_LOG(pmapxtabhist, 839 " recursing %jd", i, 0, 0, 0); 840 pmap_segtab_release(pmap, &stb->seg_seg[i], 841 true, callback, flags, va, vinc / NSEGPG); 842 KASSERT(stb->seg_seg[i] == NULL); 843 } 844 continue; 845 } 846 #endif 847 KASSERT(vinc == NBSEG); 848 849 /* get pointer to segment map */ 850 pmap_ptpage_t *ppg = stb->seg_ppg[i]; 851 if (ppg == NULL) 852 continue; 853 854 /* 855 * If our caller wants a callback, do so. 856 */ 857 if (callback != NULL) { 858 (*callback)(pmap, va, va + vinc, ppg->ppg_ptes, flags); 859 } 860 pmap_ptpage_free(pmap, ppg, __func__); 861 stb->seg_ppg[i] = NULL; 862 UVMHIST_LOG(pmapxtabhist, " zeroing tab[%jd]", i, 0, 0, 0); 863 } 864 865 if (free_stb) { 866 pmap_check_stb(stb, __func__, 867 vinc == NBSEG ? "release seg" : "release xseg"); 868 869 const vaddr_t kva = (vaddr_t)stb; 870 UVMHIST_LOG(pmapxtabhist, "about to detach", 0, 0, 0, 0); 871 pmap_page_detach(pmap, &pmap->pm_segtab_list, kva); 872 pmap_segtab_free(stb); 873 *stb_p = NULL; 874 } 875 } 876 #endif 877 878 879 880 /* 881 * Allocate the top segment table for the pmap. 882 */ 883 void 884 pmap_segtab_init(pmap_t pmap) 885 { 886 UVMHIST_FUNC(__func__); 887 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx", (uintptr_t)pmap, 0, 0, 0); 888 889 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 890 /* 891 * Constantly converting from extracted PA to VA is somewhat expensive 892 * for systems with hardware page walkers and without an inexpensive 893 * way to access arbitrary virtual addresses, so we allocate an extra 894 * root segtab so that it can contain non-virtual addresses. 895 */ 896 pmap->pm_segtab = pmap_segtab_alloc(pmap); 897 #endif 898 #if defined(PMAP_HWPAGEWALKER) 899 pmap->pm_pdetab = pmap_pdetab_alloc(pmap); 900 pmap_md_pdetab_init(pmap); 901 #endif 902 } 903 904 /* 905 * Retire the given physical map from service. 906 * Should only be called if the map contains 907 * no valid mappings. 908 */ 909 void 910 pmap_segtab_destroy(pmap_t pmap, pte_callback_t func, uintptr_t flags) 911 { 912 KASSERT(pmap != pmap_kernel()); 913 #ifdef _LP64 914 const vsize_t vinc = NBXSEG; 915 #else 916 const vsize_t vinc = NBSEG; 917 #endif 918 919 #if defined(PMAP_HWPAGEWALKER) 920 if (pmap->pm_pdetab != NULL) { 921 pmap_md_pdetab_fini(pmap); 922 pmap_pdetab_release(pmap, &pmap->pm_pdetab, 923 true, pmap->pm_minaddr, vinc); 924 } 925 #endif 926 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 927 if (pmap->pm_segtab != NULL) { 928 pmap_segtab_release(pmap, &pmap->pm_segtab, 929 func == NULL, func, flags, pmap->pm_minaddr, vinc); 930 } 931 #endif 932 933 #if defined(PMAP_HWPAGEWALKER) 934 #if !defined(PMAP_MAP_PDETABPAGE) 935 KASSERT((pmap->pm_segtab == NULL) == (pmap->pm_pdetab == NULL)); 936 #endif 937 KASSERT(pmap->pm_pdetab == NULL); 938 #endif 939 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 940 KASSERT(pmap->pm_segtab == NULL); 941 #endif 942 943 } 944 945 /* 946 * Make a new pmap (vmspace) active for the given process. 947 */ 948 void 949 pmap_segtab_activate(struct pmap *pm, struct lwp *l) 950 { 951 if (l == curlwp) { 952 KASSERTMSG(pm == l->l_proc->p_vmspace->vm_map.pmap, 953 "pm %p vs %p", pm, 954 l->l_proc->p_vmspace->vm_map.pmap); 955 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 956 struct cpu_info * const ci = l->l_cpu; 957 if (pm == pmap_kernel()) { 958 ci->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS; 959 #ifdef _LP64 960 ci->ci_pmap_user_seg0tab = PMAP_INVALID_SEGTAB_ADDRESS; 961 #endif 962 } else { 963 ci->ci_pmap_user_segtab = pm->pm_segtab; 964 #ifdef _LP64 965 ci->ci_pmap_user_seg0tab = pm->pm_segtab->seg_seg[0]; 966 #endif 967 } 968 #endif 969 } 970 } 971 972 void 973 pmap_segtab_deactivate(pmap_t pm) 974 { 975 #if !defined(PMAP_HWPAGEWALKER) || !defined(PMAP_MAP_PDETABPAGE) 976 curcpu()->ci_pmap_user_segtab = PMAP_INVALID_SEGTAB_ADDRESS; 977 #ifdef _LP64 978 curcpu()->ci_pmap_user_seg0tab = NULL; 979 #endif 980 #endif 981 } 982 983 /* 984 * Act on the given range of addresses from the specified map. 985 * 986 * It is assumed that the start and end are properly rounded to 987 * the page size. 988 */ 989 void 990 pmap_pte_process(pmap_t pmap, vaddr_t sva, vaddr_t eva, 991 pte_callback_t callback, uintptr_t flags) 992 { 993 #if 0 994 printf("%s: %p, %"PRIxVADDR", %"PRIxVADDR", %p, %"PRIxPTR"\n", 995 __func__, pmap, sva, eva, callback, flags); 996 #endif 997 while (sva < eva) { 998 vaddr_t lastseg_va = pmap_trunc_seg(sva) + NBSEG; 999 if (lastseg_va == 0 || lastseg_va > eva) 1000 lastseg_va = eva; 1001 1002 /* 1003 * If VA belongs to an unallocated segment, 1004 * skip to the next segment boundary. 1005 */ 1006 pt_entry_t * const ptep = pmap_pte_lookup(pmap, sva); 1007 if (ptep != NULL) { 1008 /* 1009 * Callback to deal with the ptes for this segment. 1010 */ 1011 (*callback)(pmap, sva, lastseg_va, ptep, flags); 1012 } 1013 /* 1014 * In theory we could release pages with no entries, 1015 * but that takes more effort than we want here. 1016 */ 1017 sva = lastseg_va; 1018 } 1019 } 1020 1021 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 1022 static pd_entry_t * 1023 pmap_pdetab_reserve(struct pmap *pmap, vaddr_t va) 1024 #elif defined(PMAP_HWPAGEWALKER) 1025 static pmap_ptpage_t ** 1026 pmap_segtab_reserve(struct pmap *pmap, vaddr_t va, pd_entry_t **pde_p) 1027 #else 1028 static pmap_ptpage_t ** 1029 pmap_segtab_reserve(struct pmap *pmap, vaddr_t va) 1030 #endif 1031 { 1032 UVMHIST_FUNC(__func__); 1033 UVMHIST_CALLARGS(pmapxtabhist, "pm %#jx va %#jx", (uintptr_t)pmap, 1034 (uintptr_t)va, 0, 0); 1035 1036 #if defined(PMAP_HWPAGEWALKER) 1037 pmap_pdetab_t *ptb = pmap->pm_pdetab; 1038 UVMHIST_LOG(pmapxtabhist, "pm_pdetab %#jx", (uintptr_t)ptb, 0, 0, 0); 1039 #endif 1040 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 1041 vaddr_t segtab_mask = PMAP_PDETABSIZE - 1; 1042 #ifdef _LP64 1043 for (size_t segshift = XSEGSHIFT; 1044 segshift > SEGSHIFT; 1045 segshift -= PGSHIFT - 3, segtab_mask = NSEGPG - 1) { 1046 pd_entry_t * const pde_p = 1047 &ptb->pde_pde[(va >> segshift) & segtab_mask]; 1048 pd_entry_t opde = atomic_load_relaxed(pde_p); 1049 1050 UVMHIST_LOG(pmapxtabhist, 1051 "ptb %#jx segshift %jd pde_p %#jx opde %#jx", 1052 ptb, segshift, pde_p, opde); 1053 1054 if (__predict_false(!pte_pde_valid_p(opde))) { 1055 ptb = pmap_pdetab_alloc(pmap); 1056 pd_entry_t npde = pte_pde_pdetab( 1057 pmap_md_direct_mapped_vaddr_to_paddr((vaddr_t)ptb), 1058 pmap == pmap_kernel()); 1059 opde = pte_pde_cas(pde_p, opde, npde); 1060 if (__predict_false(pte_pde_valid_p(opde))) { 1061 const vaddr_t kva = (vaddr_t)ptb; 1062 UVMHIST_LOG(pmapxtabhist, "about to detach", 1063 0, 0, 0, 0); 1064 pmap_page_detach(pmap, &pmap->pm_pdetab_list, 1065 kva); 1066 pmap_pdetab_free(ptb); 1067 } else { 1068 opde = npde; 1069 } 1070 } 1071 ptb = pmap_pde_to_pdetab(opde); 1072 UVMHIST_LOG(pmapxtabhist, "opde %#jx ptb %#jx", opde, ptb, 0, 0); 1073 } 1074 #elif defined(XSEGSHIFT) 1075 size_t segshift = XSEGSHIFT; 1076 1077 pd_entry_t opde = ptb->pde_pde[(va >> segshift) & segtab_mask]; 1078 KASSERT(pte_pde_valid_p(opde)); 1079 ptb = pmap_pde_to_pdetab(opde); 1080 segtab_mask = NSEGPG - 1; 1081 #endif /* _LP64 */ 1082 const size_t idx = (va >> SEGSHIFT) & segtab_mask; 1083 1084 UVMHIST_LOG(pmapxtabhist, "... returning %#jx (idx %jd)", 1085 (uintptr_t)&ptb->pde_pde[idx], idx, 0, 0); 1086 1087 return &ptb->pde_pde[idx]; 1088 #else /* PMAP_HWPAGEWALKER && PMAP_MAP_PDETABPAGE */ 1089 pmap_segtab_t *stb = pmap->pm_segtab; 1090 vaddr_t segtab_mask = PMAP_SEGTABSIZE - 1; 1091 #ifdef _LP64 1092 for (size_t segshift = XSEGSHIFT; 1093 segshift > SEGSHIFT; 1094 segshift -= PGSHIFT - 3, segtab_mask = NSEGPG - 1) { 1095 size_t idx = (va >> segshift) & segtab_mask; 1096 pmap_segtab_t ** const stb_p = &stb->seg_seg[idx]; 1097 #if defined(PMAP_HWPAGEWALKER) 1098 pmap_pdetab_t ** const ptb_p = &ptb->pde_pde[idx]; 1099 #endif /* PMAP_HWPAGEWALKER */ 1100 if (__predict_false((stb = *stb_p) == NULL)) { 1101 stb = pmap_segtab_alloc(pmap); 1102 #ifdef MULTIPROCESSOR 1103 pmap_segtab_t *ostb = atomic_cas_ptr(stb_p, NULL, stb); 1104 if (__predict_false(ostb != NULL)) { 1105 const vaddr_t kva = (vaddr_t)stb; 1106 UVMHIST_LOG(pmapxtabhist, "about to detach", 1107 0, 0, 0, 0); 1108 pmap_page_detach(pmap, &pmap->pm_segtab_list, 1109 kva); 1110 pmap_segtab_free(stb); 1111 stb = ostb; 1112 } 1113 #else 1114 *stb_p = stb; 1115 #endif /* MULTIPROCESSOR */ 1116 } 1117 } 1118 #elif defined(PMAP_HWPAGEWALKER) 1119 pmap_segtab_t opde = ptb->pde_pde[(va >> segshift) & segtab_mask]; 1120 KASSERT(pte_pde_valid_p(opde)); 1121 ptb = pmap_pde_to_pdetab(opde); 1122 segtab_mask = NSEGPG - 1; 1123 1124 #endif /* _LP64 */ 1125 size_t idx = (va >> SEGSHIFT) & segtab_mask; 1126 #if defined(PMAP_HWPAGEWALKER) 1127 #if defined(XSEGSHIFT) && (XSEGSHIFT != SEGSHIFT) 1128 *pte_p = &pmap->pm_segtab 1129 #else /* XSEGSHIFT */ 1130 *pde_p = &ptb->pde_pde[idx]; 1131 #endif /* XSEGSHIFT */ 1132 #endif /* PMAP_HWPAGEWALKER */ 1133 return &stb->seg_ppg[idx]; 1134 #endif 1135 } 1136 1137 1138 /* 1139 * Return a pointer for the pte that corresponds to the specified virtual 1140 * address (va) in the target physical map, allocating if needed. 1141 */ 1142 pt_entry_t * 1143 pmap_pte_reserve(pmap_t pmap, vaddr_t va, int flags) 1144 { 1145 UVMHIST_FUNC(__func__); 1146 UVMHIST_CALLARGS(pmapxtabhist, "pm=%#jx va=%#jx flags=%#jx", 1147 (uintptr_t)pmap, (uintptr_t)va, flags, 0); 1148 pmap_ptpage_t *ppg; 1149 paddr_t pa = 0; 1150 1151 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 1152 pd_entry_t * const pde_p = pmap_pdetab_reserve(pmap, va); 1153 ppg = pmap_pde_to_ptpage(atomic_load_relaxed(pde_p)); 1154 #elif defined(PMAP_HWPAGEWALKER) 1155 pd_entry_t *pde_p; 1156 pmap_ptpage_t ** const ppg_p = pmap_segtab_reserve(pmap, va, &pde_p); 1157 ppg = *ppg_p; 1158 #else 1159 pmap_ptpage_t ** const ppg_p = pmap_segtab_reserve(pmap, va); 1160 ppg = *ppg_p; 1161 #endif 1162 1163 if (__predict_false(ppg == NULL)) { 1164 ppg = pmap_ptpage_alloc(pmap, flags, &pa); 1165 if (__predict_false(ppg == NULL)) 1166 return NULL; 1167 1168 #if defined(PMAP_HWPAGEWALKER) 1169 pd_entry_t npde = pte_pde_ptpage(pa, pmap == pmap_kernel()); 1170 #endif 1171 #if defined(PMAP_HWPAGEWALKER) && defined(PMAP_MAP_PDETABPAGE) 1172 pd_entry_t opde = pte_pde_cas(pde_p, pte_invalid_pde(), npde); 1173 if (__predict_false(pte_pde_valid_p(opde))) { 1174 pmap_ptpage_free(pmap, ppg, __func__); 1175 ppg = pmap_pde_to_ptpage(opde); 1176 } 1177 #else 1178 #ifdef MULTIPROCESSOR 1179 pmap_ptpage_t *oppg = atomic_cas_ptr(ppg_p, NULL, ppg); 1180 /* 1181 * If another thread allocated the segtab needed for this va 1182 * free the page we just allocated. 1183 */ 1184 if (__predict_false(oppg != NULL)) { 1185 pmap_ptpage_free(pmap, ppg, __func__); 1186 ppg = oppg; 1187 #if defined(PMAP_HWPAGEWALKER) 1188 } else { 1189 pte_pde_set(pde_p, npde); 1190 #endif 1191 } 1192 #else /* !MULTIPROCESSOR */ 1193 *ppg_p = ppg; 1194 #endif /* MULTIPROCESSOR */ 1195 #endif /* PMAP_HWPAGEWALKER && PMAP_MAP_PDETABPAGE */ 1196 } 1197 1198 const size_t pte_idx = (va >> PGSHIFT) & (NPTEPG - 1); 1199 1200 return ppg->ppg_ptes + pte_idx; 1201 } 1202
Indexes created Sun Oct 25 00:26:45 UTC 2026