1 /* $NetBSD: pmap_machdep.c,v 1.29 2026/09/29 18:46:38 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2014, 2019, 2021 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas (of 3am Software Foundry), Maxime Villard, and 9 * Nick Hudson. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include "opt_riscv_debug.h" 34 #include "opt_multiprocessor.h" 35 36 #define __PMAP_PRIVATE 37 38 #include <sys/cdefs.h> 39 __RCSID("$NetBSD: pmap_machdep.c,v 1.29 2026/09/29 18:46:38 skrll Exp $"); 40 41 #include <sys/param.h> 42 #include <sys/buf.h> 43 #include <sys/cpu.h> 44 45 #include <uvm/uvm.h> 46 47 #include <riscv/machdep.h> 48 #include <riscv/sbi.h> 49 #include <riscv/sysreg.h> 50 51 #ifdef VERBOSE_INIT_RISCV 52 #define VPRINTF(...) printf(__VA_ARGS__) 53 #else 54 #define VPRINTF(...) __nothing 55 #endif 56 57 vaddr_t pmap_direct_base __read_mostly; 58 vaddr_t pmap_direct_end __read_mostly; 59 60 #ifdef _LP64 61 static pt_entry_t pmap_pte_pbmt_mask __read_mostly; 62 static pt_entry_t pmap_pte_pma __read_mostly; 63 static pt_entry_t pmap_pte_nc __read_mostly; 64 static pt_entry_t pmap_pte_io __read_mostly; 65 #endif 66 67 void 68 pmap_zero_page(paddr_t pa) 69 { 70 #ifdef _LP64 71 #ifdef PMAP_DIRECT_MAP 72 memset((void *)PMAP_DIRECT_MAP(pa), 0, PAGE_SIZE); 73 #else 74 #error "no direct map" 75 #endif 76 #else 77 KASSERT(false); 78 #endif 79 } 80 81 void 82 pmap_copy_page(paddr_t src, paddr_t dst) 83 { 84 #ifdef _LP64 85 #ifdef PMAP_DIRECT_MAP 86 memcpy((void *)PMAP_DIRECT_MAP(dst), (const void *)PMAP_DIRECT_MAP(src), 87 PAGE_SIZE); 88 #else 89 #error "no direct map" 90 #endif 91 #else 92 KASSERT(false); 93 #endif 94 } 95 96 #ifdef _LP64 97 pt_entry_t 98 pte_enter_flags_to_pbmt(int flags) 99 { 100 if (flags & PMAP_DEV) { 101 return pmap_pte_io; 102 } else if (flags & PMAP_NOCACHE) { 103 return pmap_pte_nc; 104 } 105 return pmap_pte_pma; 106 } 107 #endif 108 109 struct vm_page * 110 pmap_md_alloc_poolpage(int flags) 111 { 112 113 return uvm_pagealloc(NULL, 0, NULL, flags); 114 } 115 116 vaddr_t 117 pmap_md_map_poolpage(paddr_t pa, vsize_t len) 118 { 119 #ifdef _LP64 120 return PMAP_DIRECT_MAP(pa); 121 #else 122 panic("not supported"); 123 #endif 124 } 125 126 void 127 pmap_md_unmap_poolpage(vaddr_t pa, vsize_t len) 128 { 129 /* nothing to do */ 130 } 131 132 133 bool 134 pmap_md_direct_mapped_vaddr_p(vaddr_t va) 135 { 136 #ifdef _LP64 137 return RISCV_DIRECTMAP_P(va); 138 #else 139 return false; 140 #endif 141 } 142 143 bool 144 pmap_md_io_vaddr_p(vaddr_t va) 145 { 146 return false; 147 } 148 149 paddr_t 150 pmap_md_direct_mapped_vaddr_to_paddr(vaddr_t va) 151 { 152 #ifdef _LP64 153 #ifdef PMAP_DIRECT_MAP 154 return PMAP_DIRECT_UNMAP(va); 155 #else 156 KASSERT(false); 157 return 0; 158 #endif 159 #else 160 KASSERT(false); 161 return 0; 162 #endif 163 } 164 165 vaddr_t 166 pmap_md_direct_map_paddr(paddr_t pa) 167 { 168 #ifdef _LP64 169 return PMAP_DIRECT_MAP(pa); 170 #else 171 panic("not supported"); 172 #endif 173 } 174 175 void 176 pmap_md_init(void) 177 { 178 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 179 } 180 181 bool 182 pmap_md_ok_to_steal_p(const uvm_physseg_t bank, size_t npgs) 183 { 184 return true; 185 } 186 187 #ifdef MULTIPROCESSOR 188 void 189 pmap_md_tlb_info_attach(struct pmap_tlb_info *ti, struct cpu_info *ci) 190 { 191 } 192 #endif 193 194 195 void 196 pmap_md_pdetab_init(struct pmap *pmap) 197 { 198 KASSERT(pmap != NULL); 199 200 const vaddr_t pdetabva = (vaddr_t)pmap->pm_pdetab; 201 const paddr_t pdetabpa = pmap_md_direct_mapped_vaddr_to_paddr(pdetabva); 202 pmap->pm_md.md_ppn = pdetabpa >> PAGE_SHIFT; 203 204 /* XXXSB can we "pre-optimise" this by keeping a list of pdes to copy? */ 205 /* XXXSB for relatively normal size memory (8gb) we only need 10-20ish ptes? */ 206 /* XXXSB most (all?) of these ptes are in two consecutive ranges. */ 207 for (size_t i = NPDEPG / 2; i < NPDEPG; ++i) { 208 /* 209 * XXXSB where/when do new entries in pmap_kernel()->pm_pdetab 210 * XXXSB get added to existing pmaps? 211 * 212 * pmap_growkernel doesn't have support for fixing up exiting 213 * pmaps. (yet) 214 * 215 * Various options: 216 * 217 * - do the x86 thing. maintain a list of pmaps and update them 218 * all in pmap_growkernel. 219 * - make sure the top level entries are populated and them simply 220 * copy "them all" here. If pmap_growkernel runs the new entries 221 * will become visible to all pmaps. 222 * - ... 223 */ 224 225 /* XXXSB is this any faster than blindly copying all "high" entries? */ 226 pd_entry_t pde = pmap_kernel()->pm_pdetab->pde_pde[i]; 227 228 /* we might have leaf entries (direct map) as well as non-leaf */ 229 if (pde) { 230 pmap->pm_pdetab->pde_pde[i] = pde; 231 } 232 } 233 } 234 235 void 236 pmap_md_pdetab_fini(struct pmap *pmap) 237 { 238 239 if (pmap == pmap_kernel()) 240 return; 241 for (size_t i = NPDEPG / 2; i < NPDEPG; ++i) { 242 KASSERT(pte_invalid_pde() == 0); 243 pmap->pm_pdetab->pde_pde[i] = 0; 244 } 245 } 246 247 static void 248 pmap_md_grow(pmap_pdetab_t *ptb, vaddr_t va, vsize_t vshift, 249 vsize_t *remaining) 250 { 251 KASSERT((va & (NBSEG - 1)) == 0); 252 #ifdef _LP64 253 const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1; 254 const vsize_t vinc = 1UL << vshift; 255 256 for (size_t i = (va >> vshift) & pdetab_mask; 257 i < PMAP_PDETABSIZE; i++, va += vinc) { 258 pd_entry_t * const pde_p = 259 &ptb->pde_pde[(va >> vshift) & pdetab_mask]; 260 261 vaddr_t pdeva; 262 if (pte_pde_valid_p(*pde_p)) { 263 const paddr_t pa = pte_pde_to_paddr(*pde_p); 264 pdeva = pmap_md_direct_map_paddr(pa); 265 } else { 266 /* 267 * uvm_pageboot_alloc() returns a direct mapped address 268 */ 269 pdeva = uvm_pageboot_alloc(PAGE_SIZE); 270 paddr_t pdepa = RISCV_KVA_TO_PA(pdeva); 271 *pde_p = pte_pde_pdetab(pdepa, true); 272 memset((void *)pdeva, 0, PAGE_SIZE); 273 } 274 275 if (vshift > SEGSHIFT) { 276 pmap_md_grow((pmap_pdetab_t *)pdeva, va, 277 vshift - SEGLENGTH, remaining); 278 } else { 279 if (*remaining > vinc) 280 *remaining -= vinc; 281 else 282 *remaining = 0; 283 } 284 if (*remaining == 0) 285 return; 286 } 287 #endif 288 } 289 290 void 291 pmap_pte_xmae(void) 292 { 293 #ifdef _LP64 294 pmap_pte_pbmt_mask = PTE_XMAE; 295 pmap_pte_pma = PTE_XMAE_PMA; 296 pmap_pte_nc = PTE_XMAE_NC; 297 pmap_pte_io = PTE_XMAE_IO; 298 #endif 299 } 300 301 void 302 pmap_bootstrap(vaddr_t vstart, vaddr_t vend) 303 { 304 extern pmap_pdetab_t bootstrap_pde[PAGE_SIZE / sizeof(pd_entry_t)]; 305 306 // pmap_pdetab_t * const kptb = &pmap_kern_pdetab; 307 pmap_t pm = pmap_kernel(); 308 309 VPRINTF("common "); 310 pmap_bootstrap_common(); 311 312 VPRINTF("bs_pde %p ", bootstrap_pde); 313 314 // kend = (kend + 0x200000 - 1) & -0x200000; 315 316 /* Use the tables we already built in init_riscv() */ 317 pm->pm_pdetab = bootstrap_pde; 318 319 /* Get the PPN for our page table root */ 320 pm->pm_md.md_ppn = atop(KERN_VTOPHYS((vaddr_t)bootstrap_pde)); 321 322 /* Setup basic info like pagesize=PAGE_SIZE */ 323 // uvm_md_init(); 324 325 /* init the lock */ 326 // XXXNH per cpu? 327 pmap_tlb_info_init(&pmap_tlb0_info); 328 329 VPRINTF("ASID max %x ", pmap_tlb0_info.ti_asid_max); 330 331 #ifdef MULTIPROCESSOR 332 VPRINTF("kcpusets "); 333 334 kcpuset_create(&pm->pm_onproc, true); 335 kcpuset_create(&pm->pm_active, true); 336 KASSERT(pm->pm_onproc != NULL); 337 KASSERT(pm->pm_active != NULL); 338 339 struct cpu_info * const ci = curcpu(); 340 kcpuset_set(pm->pm_onproc, cpu_index(ci)); 341 kcpuset_set(pm->pm_active, cpu_index(ci)); 342 #endif 343 344 VPRINTF("nkmempages "); 345 /* 346 * Compute the number of pages kmem_arena will have. This will also 347 * be called by uvm_km_bootstrap later, but that doesn't matter 348 */ 349 kmeminit_nkmempages(); 350 351 /* Get size of buffer cache and set an upper limit */ 352 buf_setvalimit((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS) / 8); 353 vsize_t bufsz = buf_memcalc(); 354 buf_setvalimit(bufsz); 355 356 vsize_t kvmsize = (VM_PHYS_SIZE + (ubc_nwins << ubc_winshift) + 357 bufsz + 16 * NCARGS + pager_map_size) + 358 /*(maxproc * UPAGES) + */nkmempages * NBPG; 359 360 #ifdef SYSVSHM 361 kvmsize += shminfo.shmall; 362 #endif 363 364 /* Calculate VA address space and roundup to NBSEG tables */ 365 kvmsize = roundup(kvmsize, NBSEG); 366 367 /* 368 * Initialize `FYI' variables. Note we're relying on 369 * the fact that BSEARCH sorts the vm_physmem[] array 370 * for us. Must do this before uvm_pageboot_alloc() 371 * can be called. 372 */ 373 pmap_limits.avail_start = ptoa(uvm_physseg_get_start(uvm_physseg_get_first())); 374 pmap_limits.avail_end = ptoa(uvm_physseg_get_end(uvm_physseg_get_last())); 375 376 /* 377 * Update the naive settings in pmap_limits to the actual KVA range. 378 */ 379 pmap_limits.virtual_start = vstart; 380 pmap_limits.virtual_end = vend; 381 382 VPRINTF("limits: %" PRIxVADDR " - %" PRIxVADDR "\n", vstart, vend); 383 384 const vaddr_t kvmstart = vstart; 385 pmap_curmaxkvaddr = vstart + kvmsize; 386 387 VPRINTF("kva : %" PRIxVADDR " - %" PRIxVADDR "\n", kvmstart, 388 pmap_curmaxkvaddr); 389 390 pmap_md_grow(pmap_kernel()->pm_pdetab, kvmstart, XSEGSHIFT, &kvmsize); 391 392 /* 393 * Initialize the pools. 394 */ 395 pool_init(&pmap_pmap_pool, PMAP_SIZE, 0, 0, 0, "pmappl", 396 &pool_allocator_nointr, IPL_NONE); 397 398 pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, 0, 0, "pvpl", 399 #ifdef KASAN 400 NULL, 401 #else 402 &pmap_pv_page_allocator, 403 #endif 404 IPL_NONE); 405 406 // riscv_dcache_align 407 pmap_pvlist_lock_init(CACHE_LINE_SIZE); 408 } 409 410 411 vsize_t 412 pmap_kenter_range(vaddr_t va, paddr_t pa, vsize_t size, 413 vm_prot_t prot, u_int flags) 414 { 415 extern pd_entry_t l1_pte[PAGE_SIZE / sizeof(pd_entry_t)]; 416 417 vaddr_t sva = MEGAPAGE_TRUNC(va); 418 paddr_t spa = MEGAPAGE_TRUNC(pa); 419 const vaddr_t eva = MEGAPAGE_ROUND(va + size); 420 const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1; 421 const vsize_t vshift = SEGSHIFT; 422 const pt_entry_t pbmt_flag = pte_enter_flags_to_pbmt(flags); 423 424 while (sva < eva) { 425 const size_t sidx = (sva >> vshift) & pdetab_mask; 426 427 l1_pte[sidx] = PA_TO_PTE(spa) | PTE_KERN | PTE_HARDWIRED | 428 PTE_RW | pbmt_flag; 429 spa += NBSEG; 430 sva += NBSEG; 431 } 432 433 return 0; 434 } 435
Indexes created Sun Oct 25 00:26:45 UTC 2026