1 /* $NetBSD: riscv_machdep.c,v 1.51 2026/07/25 15:49:49 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2014, 2019, 2022 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry, and by Nick Hudson. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_ddb.h" 33 #include "opt_modular.h" 34 #include "opt_multiprocessor.h" 35 #include "opt_riscv_debug.h" 36 37 #include <sys/cdefs.h> 38 __RCSID("$NetBSD: riscv_machdep.c,v 1.51 2026/07/25 15:49:49 skrll Exp $"); 39 40 #include <sys/param.h> 41 42 #include <sys/asan.h> 43 #include <sys/boot_flag.h> 44 #include <sys/cpu.h> 45 #include <sys/exec.h> 46 #include <sys/kmem.h> 47 #include <sys/ktrace.h> 48 #include <sys/lwp.h> 49 #include <sys/module.h> 50 #include <sys/mount.h> 51 #include <sys/msgbuf.h> 52 #include <sys/optstr.h> 53 #include <sys/proc.h> 54 #include <sys/reboot.h> 55 #include <sys/syscall.h> 56 #include <sys/sysctl.h> 57 #include <sys/systm.h> 58 59 #include <dev/cons.h> 60 #ifdef __HAVE_MM_MD_KERNACC 61 #include <dev/mm.h> 62 #endif 63 64 #include <uvm/uvm_extern.h> 65 66 #include <riscv/cpufunc.h> 67 #include <riscv/frame.h> 68 #include <riscv/locore.h> 69 #include <riscv/machdep.h> 70 #include <riscv/pte.h> 71 #include <riscv/sbi.h> 72 #include <riscv/userret.h> 73 74 #include <libfdt.h> 75 #include <dev/fdt/fdtvar.h> 76 #include <dev/fdt/fdt_boot.h> 77 #include <dev/fdt/fdt_console.h> 78 #include <dev/fdt/fdt_memory.h> 79 #include <dev/fdt/fdt_private.h> 80 #include <dev/fdt/fdt_platform.h> 81 82 int cpu_printfataltraps = 1; 83 char machine[] = MACHINE; 84 char machine_arch[] = MACHINE_ARCH; 85 86 #ifdef VERBOSE_INIT_RISCV 87 #define VPRINTF(...) printf(__VA_ARGS__) 88 #else 89 #define VPRINTF(...) __nothing 90 #endif 91 92 /* 64 should be enough, even for a ZFS UUID */ 93 #define MAX_BOOT_DEV_STR 64 94 95 char bootdevstr[MAX_BOOT_DEV_STR] = ""; 96 char *boot_args = NULL; 97 98 paddr_t physical_start; 99 paddr_t physical_end; 100 101 static void 102 earlyconsputc(dev_t dev, int c) 103 { 104 uartputc(c); 105 } 106 107 static int 108 earlyconsgetc(dev_t dev) 109 { 110 return uartgetc(); 111 } 112 113 static struct consdev earlycons = { 114 .cn_putc = earlyconsputc, 115 .cn_getc = earlyconsgetc, 116 .cn_pollc = nullcnpollc, 117 }; 118 119 struct vm_map *phys_map; 120 121 struct trapframe cpu_ddb_regs; 122 const pcu_ops_t * const pcu_ops_md_defs[PCU_UNIT_COUNT] = { 123 #ifdef FPE 124 [PCU_FPU] = &pcu_fpu_ops, 125 #endif 126 }; 127 128 /* 129 * Used by PHYSTOV and VTOPHYS -- Will be set be BSS is zeroed so 130 * keep it in data 131 */ 132 unsigned long kern_vtopdiff __attribute__((__section__(".data"))); 133 134 135 /* 136 * machine dependent system variables. 137 */ 138 SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup") 139 { 140 sysctl_createv(clog, 0, NULL, NULL, 141 CTLFLAG_PERMANENT, 142 CTLTYPE_NODE, "machdep", NULL, 143 NULL, 0, NULL, 0, 144 CTL_MACHDEP, CTL_EOL); 145 } 146 147 /* 148 * Set registers on exec. 149 * Clear all registers except sp, pc. 150 * sp is set to the stack pointer passed in. pc is set to the entry 151 * point given by the exec_package passed in. 152 */ 153 void 154 setregs(struct lwp *l, struct exec_package *pack, vaddr_t stack) 155 { 156 struct trapframe * const tf = l->l_md.md_utf; 157 struct proc * const p = l->l_proc; 158 159 memset(tf, 0, sizeof(*tf)); 160 tf->tf_sp = (intptr_t)stack_align(stack); 161 tf->tf_pc = (intptr_t)pack->ep_entry & ~1; 162 #ifdef _LP64 163 tf->tf_sr = (p->p_flag & PK_32) ? SR_USER32 : SR_USER64; 164 #else 165 tf->tf_sr = SR_USER; 166 #endif 167 168 // Set up arguments for ___start(cleanup, ps_strings) 169 tf->tf_a0 = 0; // cleanup 170 tf->tf_a1 = p->p_psstrp; // ps_strings 171 172 /* 173 * Must have interrupts disabled for exception return. 174 * Must be switching to user mode. 175 * Must enable interrupts after sret. 176 */ 177 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SIE) == 0); 178 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SPP) == 0); 179 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SPIE) != 0); 180 } 181 182 void 183 md_child_return(struct lwp *l) 184 { 185 struct trapframe * const tf = lwp_trapframe(l); 186 187 tf->tf_a0 = 0; 188 tf->tf_a1 = 1; 189 #ifdef FPE 190 /* Disable FP as we can't be using it (yet). */ 191 tf->tf_sr &= ~SR_FS; 192 #endif 193 194 /* 195 * Must have interrupts disabled for exception return. 196 * Must be switching to user mode. 197 * Must enable interrupts after sret. 198 */ 199 200 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SIE) == 0); 201 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SPP) == 0); 202 KASSERT(__SHIFTOUT(tf->tf_sr, SR_SPIE) != 0); 203 204 userret(l); 205 } 206 207 /* 208 * Process the tail end of a posix_spawn() for the child. 209 */ 210 void 211 cpu_spawn_return(struct lwp *l) 212 { 213 userret(l); 214 } 215 216 /* 217 * Start a new LWP 218 */ 219 void 220 startlwp(void *arg) 221 { 222 ucontext_t * const uc = arg; 223 lwp_t * const l = curlwp; 224 int error __diagused; 225 226 error = cpu_setmcontext(l, &uc->uc_mcontext, uc->uc_flags); 227 KASSERT(error == 0); 228 229 kmem_free(uc, sizeof(*uc)); 230 userret(l); 231 } 232 233 // We've worked hard to make sure struct reg and __gregset_t are the same. 234 // Ditto for struct fpreg and fregset_t. 235 236 #ifdef _LP64 237 CTASSERT(sizeof(struct reg) == sizeof(__gregset_t)); 238 #endif 239 CTASSERT(sizeof(struct fpreg) == sizeof(__fregset_t)); 240 241 void 242 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 243 { 244 const struct trapframe * const tf = l->l_md.md_utf; 245 246 /* Save register context. */ 247 *(struct reg *)mcp->__gregs = tf->tf_regs; 248 249 *flags |= _UC_CPU | _UC_TLSBASE; 250 251 /* Save floating point register context, if any. */ 252 KASSERT(l == curlwp); 253 if (fpu_valid_p(l)) { 254 /* 255 * If this process is the current FP owner, dump its 256 * context to the PCB first. 257 */ 258 fpu_save(l); 259 260 struct pcb * const pcb = lwp_getpcb(l); 261 *(struct fpreg *)mcp->__fregs = pcb->pcb_fpregs; 262 *flags |= _UC_FPU; 263 } 264 } 265 266 int 267 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 268 { 269 /* 270 * Verify that at least the PC and SP are user addresses. 271 */ 272 if ((intptr_t) mcp->__gregs[_REG_PC] < 0 273 || (intptr_t) mcp->__gregs[_REG_SP] < 0 274 || (mcp->__gregs[_REG_PC] & 1)) 275 return EINVAL; 276 277 return 0; 278 } 279 280 int 281 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 282 { 283 struct trapframe * const tf = l->l_md.md_utf; 284 struct proc * const p = l->l_proc; 285 const __greg_t * const gr = mcp->__gregs; 286 int error; 287 288 /* Restore register context, if any. */ 289 if (flags & _UC_CPU) { 290 error = cpu_mcontext_validate(l, mcp); 291 if (error) 292 return error; 293 294 /* 295 * Avoid updating TLS register here. 296 */ 297 const __greg_t saved_tp = tf->tf_reg[_REG_TP]; 298 tf->tf_regs = *(const struct reg *)gr; 299 tf->tf_reg[_REG_TP] = saved_tp; 300 } 301 302 /* Restore the private thread context */ 303 if (flags & _UC_TLSBASE) { 304 lwp_setprivate(l, (void *)(intptr_t)mcp->__gregs[_X_TP]); 305 } 306 307 /* Restore floating point register context, if any. */ 308 if (flags & _UC_FPU) { 309 KASSERT(l == curlwp); 310 /* Tell PCU we are replacing the FPU contents. */ 311 fpu_replace(l); 312 313 /* 314 * The PCB FP regs struct includes the FP CSR, so use the 315 * proper size of fpreg when copying. 316 */ 317 struct pcb * const pcb = lwp_getpcb(l); 318 pcb->pcb_fpregs = *(const struct fpreg *)mcp->__fregs; 319 } 320 321 mutex_enter(p->p_lock); 322 if (flags & _UC_SETSTACK) 323 l->l_sigstk.ss_flags |= SS_ONSTACK; 324 if (flags & _UC_CLRSTACK) 325 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 326 mutex_exit(p->p_lock); 327 328 return 0; 329 } 330 331 void 332 cpu_need_resched(struct cpu_info *ci, struct lwp *l, int flags) 333 { 334 KASSERT(kpreempt_disabled()); 335 336 if ((flags & RESCHED_KPREEMPT) != 0) { 337 #ifdef __HAVE_PREEMPTION 338 if ((flags & RESCHED_REMOTE) != 0) { 339 cpu_send_ipi(ci, IPI_KPREEMPT); 340 } else { 341 softint_trigger(SOFTINT_KPREEMPT); 342 } 343 #endif 344 return; 345 } 346 if ((flags & RESCHED_REMOTE) != 0) { 347 #ifdef MULTIPROCESSOR 348 cpu_send_ipi(ci, IPI_AST); 349 #endif 350 } else { 351 l->l_md.md_astpending = 1; /* force call to ast() */ 352 } 353 } 354 355 void 356 cpu_signotify(struct lwp *l) 357 { 358 KASSERT(kpreempt_disabled()); 359 #ifdef __HAVE_FAST_SOFTINTS 360 KASSERT(lwp_locked(l, NULL)); 361 #endif 362 363 if (l->l_cpu != curcpu()) { 364 #ifdef MULTIPROCESSOR 365 cpu_send_ipi(l->l_cpu, IPI_AST); 366 #endif 367 } else { 368 l->l_md.md_astpending = 1; /* force call to ast() */ 369 } 370 } 371 372 void 373 cpu_need_proftick(struct lwp *l) 374 { 375 KASSERT(kpreempt_disabled()); 376 KASSERT(l->l_cpu == curcpu()); 377 378 l->l_pflag |= LP_OWEUPC; 379 l->l_md.md_astpending = 1; /* force call to ast() */ 380 } 381 382 383 /* Sync the discs, unmount the filesystems, and adjust the todr */ 384 static void 385 bootsync(void) 386 { 387 static bool bootsyncdone = false; 388 389 if (bootsyncdone) 390 return; 391 392 bootsyncdone = true; 393 394 /* Make sure we can still manage to do things */ 395 if ((csr_sstatus_read() & SR_SIE) == 0) { 396 /* 397 * If we get here then boot has been called without RB_NOSYNC 398 * and interrupts were disabled. This means the boot() call 399 * did not come from a user process e.g. shutdown, but must 400 * have come from somewhere in the kernel. 401 */ 402 ENABLE_INTERRUPTS(); 403 printf("Warning interrupts disabled during boot()\n"); 404 } 405 406 vfs_shutdown(); 407 } 408 409 410 void 411 cpu_reboot(int howto, char *bootstr) 412 { 413 414 /* 415 * If RB_NOSYNC was not specified sync the discs. 416 * Note: Unless cold is set to 1 here, syslogd will die during the 417 * unmount. It looks like syslogd is getting woken up only to find 418 * that it cannot page part of the binary in as the filesystem has 419 * been unmounted. 420 */ 421 if ((howto & RB_NOSYNC) == 0) 422 bootsync(); 423 424 #if 0 425 /* Disable interrupts. */ 426 const int s = splhigh(); 427 428 /* Do a dump if requested. */ 429 if ((howto & (RB_DUMP | RB_HALT)) == RB_DUMP) 430 dumpsys(); 431 432 splx(s); 433 #endif 434 435 pmf_system_shutdown(boothowto); 436 437 /* Say NO to interrupts for good */ 438 splhigh(); 439 440 /* Run any shutdown hooks */ 441 doshutdownhooks(); 442 443 /* Make sure IRQ's are disabled */ 444 DISABLE_INTERRUPTS(); 445 446 if (howto & RB_HALT) { 447 if (howto & RB_POWERDOWN) { 448 printf("power off...\n"); 449 sbi_system_reset(SBI_RESET_TYPE_SHUTDOWN, 450 SBI_RESET_REASON_NONE); 451 } /* FALLTHRU */ 452 printf("\n"); 453 printf("The operating system has halted.\n"); 454 printf("Please press any key to reboot.\n\n"); 455 cnpollc(true); /* for proper keyboard command handling */ 456 if (cngetc() == 0) { 457 /* no console attached, so just hlt */ 458 printf("No keyboard - cannot reboot after all.\n"); 459 goto spin; 460 } 461 cnpollc(false); 462 } 463 464 printf("rebooting...\n"); 465 466 sbi_system_reset(SBI_RESET_TYPE_COLDREBOOT, SBI_RESET_REASON_NONE); 467 spin: 468 for (;;) { 469 asm volatile("wfi" ::: "memory"); 470 } 471 /* NOTREACHED */ 472 } 473 474 void 475 cpu_dumpconf(void) 476 { 477 // TBD!! 478 } 479 480 481 int 482 cpu_lwp_setprivate(lwp_t *l, void *addr) 483 { 484 struct trapframe * const tf = lwp_trapframe(l); 485 486 tf->tf_reg[_REG_TP] = (register_t)addr; 487 488 return 0; 489 } 490 491 492 void 493 cpu_startup(void) 494 { 495 vaddr_t minaddr, maxaddr; 496 char pbuf[10]; /* "999999 MB" -- But Sv39 is max 512GB */ 497 498 /* 499 * Good {morning,afternoon,evening,night}. 500 */ 501 printf("%s%s", copyright, version); 502 format_bytes(pbuf, sizeof(pbuf), ctob(physmem)); 503 printf("total memory = %s\n", pbuf); 504 505 minaddr = 0; 506 /* 507 * Allocate a submap for physio. 508 */ 509 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 510 VM_PHYS_SIZE, 0, FALSE, NULL); 511 512 format_bytes(pbuf, sizeof(pbuf), ptoa(uvm_availmem(false))); 513 printf("avail memory = %s\n", pbuf); 514 515 #ifdef MULTIPROCESSOR 516 kcpuset_create(&cpus_halted, true); 517 KASSERT(cpus_halted != NULL); 518 519 kcpuset_create(&cpus_hatched, true); 520 KASSERT(cpus_hatched != NULL); 521 522 kcpuset_create(&cpus_paused, true); 523 KASSERT(cpus_paused != NULL); 524 525 kcpuset_create(&cpus_resumed, true); 526 KASSERT(cpus_resumed != NULL); 527 528 kcpuset_create(&cpus_running, true); 529 KASSERT(cpus_running != NULL); 530 531 kcpuset_set(cpus_hatched, cpu_index(curcpu())); 532 kcpuset_set(cpus_running, cpu_index(curcpu())); 533 #endif 534 535 fdtbus_intr_init(); 536 537 fdt_setup_rndseed(); 538 fdt_setup_efirng(); 539 } 540 541 static void 542 riscv_add_memory(const struct fdt_memory *m, void *arg) 543 { 544 paddr_t first = atop(m->start); 545 paddr_t last = atop(m->end); 546 int freelist = VM_FREELIST_DEFAULT; 547 548 VPRINTF("adding %#16" PRIxPADDR " - %#16" PRIxPADDR" to freelist %d\n", 549 m->start, m->end, freelist); 550 551 uvm_page_physload(first, last, first, last, freelist); 552 physmem += last - first; 553 } 554 555 556 static void 557 cpu_kernel_vm_init(paddr_t memory_start, paddr_t memory_end) 558 { 559 extern char __kernel_text[]; 560 extern char _end[]; 561 562 vaddr_t kernstart = trunc_page((vaddr_t)__kernel_text); 563 vaddr_t kernend = round_page((vaddr_t)_end); 564 paddr_t kernstart_phys = KERN_VTOPHYS(kernstart); 565 paddr_t kernend_phys = KERN_VTOPHYS(kernend); 566 567 VPRINTF("%s: kernel phys start %#" PRIxPADDR " end %#" PRIxPADDR "\n", 568 __func__, kernstart_phys, kernend_phys); 569 fdt_memory_remove_range(kernstart_phys, 570 kernend_phys - kernstart_phys); 571 572 #if 0 573 /* 574 * Don't give these pages to UVM. 575 * 576 * cpu_kernel_vm_init need to create proper tables then the following 577 * will be true. 578 * 579 * Now we have APs started the pages used for stacks and L1PT can 580 * be given to uvm 581 */ 582 extern char const __start__init_memory[]; 583 extern char const __stop__init_memory[] __weak; 584 if (&__start__init_memory[0] != &__stop__init_memory[0]) { 585 const paddr_t spa = KERN_VTOPHYS((vaddr_t)__start__init_memory); 586 const paddr_t epa = KERN_VTOPHYS((vaddr_t)__stop__init_memory); 587 588 VPRINTF("%s: init phys start %#" PRIxPADDR 589 " end %#" PRIxPADDR "\n", __func__, spa, epa); 590 fdt_memory_remove_range(spa, epa - spa); 591 } 592 #endif 593 594 #ifdef _LP64 595 paddr_t pa = memory_start & ~XSEGOFSET; 596 pmap_direct_base = RISCV_DIRECTMAP_START; 597 extern pd_entry_t l2_pte[PAGE_SIZE / sizeof(pd_entry_t)]; 598 599 600 const vsize_t vshift = XSEGSHIFT; 601 const vaddr_t pdetab_mask = PMAP_PDETABSIZE - 1; 602 const vsize_t inc = 1UL << vshift; 603 604 const vaddr_t sva = RISCV_DIRECTMAP_START + pa; 605 const vaddr_t eva = RISCV_DIRECTMAP_END; 606 const size_t sidx = (sva >> vshift) & pdetab_mask; 607 const size_t eidx = (eva >> vshift) & pdetab_mask; 608 609 /* Allocate gigapages covering all physical memory in the direct map. */ 610 pt_entry_t pbmt_flag = pte_enter_flags_to_pbmt(0); 611 for (size_t i = sidx; i < eidx && pa < memory_end; i++, pa += inc) { 612 l2_pte[i] = PA_TO_PTE(pa) | PTE_KERN | PTE_HARDWIRED | PTE_RW | 613 pbmt_flag; 614 VPRINTF("dm: %p : %#" PRIxPADDR "\n", &l2_pte[i], l2_pte[i]); 615 } 616 #endif 617 // pt_dump(printf); 618 } 619 620 static void 621 riscv_init_lwp0_uarea(void) 622 { 623 extern char lwp0uspace[]; 624 625 uvm_lwp_setuarea(&lwp0, (vaddr_t)lwp0uspace); 626 memset(&lwp0.l_md, 0, sizeof(lwp0.l_md)); 627 memset(lwp_getpcb(&lwp0), 0, sizeof(struct pcb)); 628 629 struct trapframe *tf = (struct trapframe *)(lwp0uspace + USPACE) - 1; 630 memset(tf, 0, sizeof(*tf)); 631 632 lwp0.l_md.md_utf = lwp0.l_md.md_ktf = tf; 633 } 634 635 636 static void 637 riscv_print_memory(const struct fdt_memory *m, void *arg) 638 { 639 640 VPRINTF("FDT /memory @ 0x%" PRIx64 " size 0x%" PRIx64 "\n", 641 m->start, m->end - m->start); 642 } 643 644 645 static void 646 parse_mi_bootargs(char *args) 647 { 648 int howto; 649 bool found, start, skipping; 650 651 if (args == NULL) 652 return; 653 654 start = true; 655 skipping = false; 656 for (char *cp = args; *cp; cp++) { 657 /* check for "words" starting with a "-" only */ 658 if (start) { 659 if (*cp == '-') { 660 skipping = false; 661 } else { 662 skipping = true; 663 } 664 start = false; 665 continue; 666 } 667 668 if (*cp == ' ') { 669 start = true; 670 skipping = false; 671 continue; 672 } 673 674 if (skipping) { 675 continue; 676 } 677 678 /* Check valid boot flags */ 679 howto = 0; 680 BOOT_FLAG(*cp, howto); 681 if (!howto) 682 printf("bootflag '%c' not recognised\n", *cp); 683 else 684 boothowto |= howto; 685 } 686 687 found = optstr_get(args, "root", bootdevstr, sizeof(bootdevstr)); 688 if (found) { 689 bootspec = bootdevstr; 690 } 691 } 692 693 694 void 695 init_riscv(register_t hartid, paddr_t dtb) 696 { 697 698 /* set temporally to work printf()/panic() even before consinit() */ 699 cn_tab = &earlycons; 700 701 /* Load FDT */ 702 const vaddr_t dtbva = VM_KERNEL_DTB_BASE + (dtb & (NBSEG - 1)); 703 void *fdt_data = (void *)dtbva; 704 int error = fdt_check_header(fdt_data); 705 if (error != 0) 706 panic("fdt_check_header failed: %s", fdt_strerror(error)); 707 708 fdtbus_init(fdt_data); 709 710 /* Lookup platform specific backend */ 711 const struct fdt_platform * const plat = fdt_platform_find(); 712 if (plat == NULL) 713 panic("Kernel does not support this device"); 714 715 /* Early console may be available, announce ourselves. */ 716 VPRINTF("FDT<%p>\n", fdt_data); 717 718 boot_args = fdt_get_bootargs(); 719 720 /* Heads up ... Setup the CPU / MMU / TLB functions. */ 721 VPRINTF("cpufunc\n"); 722 if (set_cpufuncs()) 723 panic("cpu not recognized!"); 724 725 VPRINTF("devmap %p\n", plat->fp_devmap()); 726 pmap_devmap_bootstrap(0, plat->fp_devmap()); 727 728 VPRINTF("bootstrap\n"); 729 plat->fp_bootstrap(); 730 731 /* 732 * If stdout-path is specified on the command line, override the 733 * value in /chosen/stdout-path before initializing console. 734 */ 735 VPRINTF("stdout\n"); 736 fdt_update_stdout_path(fdt_data, boot_args); 737 738 /* 739 * Done making changes to the FDT. 740 */ 741 fdt_pack(fdt_data); 742 743 const uint32_t dtbsize = round_page(fdt_totalsize(fdt_data)); 744 745 VPRINTF("fdt size %x/%x\n", dtbsize, fdt_totalsize(fdt_data)); 746 747 VPRINTF("consinit "); 748 consinit(); 749 VPRINTF("ok\n"); 750 751 /* Talk to the user */ 752 printf("NetBSD/riscv (fdt) booting ...\n"); 753 754 #ifdef BOOT_ARGS 755 char mi_bootargs[] = BOOT_ARGS; 756 parse_mi_bootargs(mi_bootargs); 757 #endif 758 759 uint64_t memory_start, memory_end; 760 fdt_memory_get(&memory_start, &memory_end); 761 physical_start = memory_start; 762 physical_end = memory_end; 763 764 fdt_memory_foreach(riscv_print_memory, NULL); 765 766 /* Cannot map memory above largest page number */ 767 const uint64_t maxppn = __SHIFTOUT_MASK(PTE_PPN) - 1; 768 const uint64_t memory_limit = ptoa(maxppn); 769 770 if (memory_end > memory_limit) { 771 fdt_memory_remove_range(memory_limit, memory_end - memory_limit); 772 memory_end = memory_limit; 773 } 774 775 uint64_t memory_size __unused = memory_end - memory_start; 776 777 VPRINTF("%s: memory start %" PRIx64 " end %" PRIx64 " (len %" 778 PRIx64 ")\n", __func__, memory_start, memory_end, memory_size); 779 780 /* Parse ramdisk, rndseed, and firmware's RNG from EFI */ 781 fdt_probe_initrd(); 782 fdt_probe_rndseed(); 783 fdt_probe_efirng(); 784 785 fdt_memory_remove_reserved(memory_start, memory_end); 786 787 fdt_memory_remove_range(dtb, dtbsize); 788 fdt_reserve_initrd(); 789 fdt_reserve_rndseed(); 790 fdt_reserve_efirng(); 791 792 /* Perform PT build and VM init */ 793 cpu_kernel_vm_init(memory_start, memory_end); 794 795 VPRINTF("bootargs: %s\n", boot_args); 796 797 parse_mi_bootargs(boot_args); 798 799 #ifdef DDB 800 if (boothowto & RB_KDB) { 801 printf("Entering DDB...\n"); 802 cpu_Debugger(); 803 } 804 #endif 805 806 extern char __kernel_text[]; 807 extern char _end[]; 808 // extern char __data_start[]; 809 // extern char __rodata_start[]; 810 811 vaddr_t kernstart = trunc_page((vaddr_t)__kernel_text); 812 vaddr_t kernend = round_page((vaddr_t)_end); 813 paddr_t kernstart_phys __unused = KERN_VTOPHYS(kernstart); 814 paddr_t kernend_phys __unused = KERN_VTOPHYS(kernend); 815 816 vaddr_t kernelvmstart; 817 818 vaddr_t kernstart_mega __unused = MEGAPAGE_TRUNC(kernstart); 819 vaddr_t kernend_mega = MEGAPAGE_ROUND(kernend); 820 821 kernelvmstart = kernend_mega; 822 823 #if 0 824 #ifdef MODULAR 825 #define MODULE_RESERVED_MAX (1024 * 1024 * 128) 826 #define MODULE_RESERVED_SIZE (1024 * 1024 * 32) /* good enough? */ 827 module_start = kernelvmstart; 828 module_end = kernend_mega + MODULE_RESERVED_SIZE; 829 if (module_end >= kernstart_mega + MODULE_RESERVED_MAX) 830 module_end = kernstart_mega + MODULE_RESERVED_MAX; 831 KASSERT(module_end > kernend_mega); 832 kernelvmstart = module_end; 833 #endif /* MODULAR */ 834 #endif 835 KASSERT(kernelvmstart < VM_KERNEL_VM_BASE); 836 837 kernelvmstart = VM_KERNEL_VM_BASE; 838 839 /* 840 * msgbuf is allocated from the top of the last biggest memory block. 841 */ 842 paddr_t msgbufaddr = 0; 843 844 #ifdef _LP64 845 /* XXX check all ranges for last one with a big enough hole */ 846 msgbufaddr = memory_end - MSGBUFSIZE; 847 KASSERT(msgbufaddr != 0); /* no space for msgbuf */ 848 fdt_memory_remove_range(msgbufaddr, MSGBUFSIZE); 849 msgbufaddr = RISCV_PA_TO_KVA(msgbufaddr); 850 VPRINTF("msgbufaddr = %#lx\n", msgbufaddr); 851 initmsgbuf((void *)msgbufaddr, MSGBUFSIZE); 852 #endif 853 854 KASSERT(msgbufaddr != 0); /* no space for msgbuf */ 855 #ifdef _LP64 856 initmsgbuf((void *)RISCV_PA_TO_KVA(msgbufaddr), MSGBUFSIZE); 857 #endif 858 859 #define DPRINTF(v) VPRINTF("%24s = 0x%16lx\n", #v, (unsigned long)v); 860 861 VPRINTF("------------------------------------------\n"); 862 DPRINTF(kern_vtopdiff); 863 DPRINTF(memory_start); 864 DPRINTF(memory_end); 865 DPRINTF(memory_size); 866 DPRINTF(kernstart_phys); 867 DPRINTF(kernend_phys) 868 DPRINTF(msgbufaddr); 869 // DPRINTF(physical_end); 870 DPRINTF(VM_MIN_KERNEL_ADDRESS); 871 DPRINTF(kernstart_mega); 872 DPRINTF(kernstart); 873 DPRINTF(kernend); 874 DPRINTF(kernend_mega); 875 #if 0 876 #ifdef MODULAR 877 DPRINTF(module_start); 878 DPRINTF(module_end); 879 #endif 880 #endif 881 DPRINTF(VM_MAX_KERNEL_ADDRESS); 882 #ifdef _LP64 883 DPRINTF(pmap_direct_base); 884 #endif 885 VPRINTF("------------------------------------------\n"); 886 887 #undef DPRINTF 888 889 uvm_md_init(); 890 891 /* 892 * pass memory pages to uvm 893 */ 894 physmem = 0; 895 fdt_memory_foreach(riscv_add_memory, NULL); 896 897 pmap_bootstrap(kernelvmstart, VM_MAX_KERNEL_ADDRESS); 898 899 kasan_init(); 900 901 /* Finish setting up lwp0 on our end before we call main() */ 902 riscv_init_lwp0_uarea(); 903 904 error = 0; 905 if ((boothowto & RB_MD1) == 0) { 906 VPRINTF("mpstart\n"); 907 if (plat->fp_mpstart) 908 error = plat->fp_mpstart(); 909 } 910 if (error) 911 printf("AP startup problems\n"); 912 } 913 914 915 #ifdef __HAVE_MM_MD_KERNACC 916 917 #define IN_RANGE_P(addr, start, end) (start) <= (addr) && (addr) < (end) 918 #ifdef _LP64 919 #define IN_DIRECTMAP_P(va) \ 920 IN_RANGE_P(va, RISCV_DIRECTMAP_START, RISCV_DIRECTMAP_END) 921 #else 922 #define IN_DIRECTMAP_P(va) false 923 #endif 924 925 int 926 mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled) 927 { 928 extern char __kernel_text[]; 929 extern char _end[]; 930 extern char __data_start[]; 931 932 const vaddr_t kernstart = trunc_page((vaddr_t)__kernel_text); 933 const vaddr_t kernend = round_page((vaddr_t)_end); 934 const vaddr_t data_start = (vaddr_t)__data_start; 935 936 const vaddr_t va = (vaddr_t)ptr; 937 938 *handled = false; 939 if (IN_RANGE_P(va, kernstart, kernend)) { 940 *handled = true; 941 if (va < data_start && (prot & VM_PROT_WRITE) != 0) { 942 return EFAULT; 943 } 944 } else if (IN_DIRECTMAP_P(va)) { 945 *handled = true; 946 } 947 948 return 0; 949 } 950 #endif 951 952 953 #ifdef _LP64 954 static void 955 pte_bits(void (*pr)(const char *, ...), pt_entry_t pte) 956 { 957 (*pr)("%c%c%c%c%c%c%c%c", 958 (pte & PTE_D) ? 'D' : '.', 959 (pte & PTE_A) ? 'A' : '.', 960 (pte & PTE_G) ? 'G' : '.', 961 (pte & PTE_U) ? 'U' : '.', 962 (pte & PTE_X) ? 'X' : '.', 963 (pte & PTE_W) ? 'W' : '.', 964 (pte & PTE_R) ? 'R' : '.', 965 (pte & PTE_V) ? 'V' : '.'); 966 } 967 968 static void 969 dump_ln_table(paddr_t pdp_pa, int topbit, int level, vaddr_t va, 970 void (*pr)(const char *, ...) __printflike(1, 2)) 971 { 972 pd_entry_t *pdp = (void *)PMAP_DIRECT_MAP(pdp_pa); 973 974 (*pr)("l%u @ pa %#16" PRIxREGISTER "\n", level, pdp_pa); 975 for (size_t i = 0; i < PAGE_SIZE / sizeof(pd_entry_t); i++) { 976 pd_entry_t entry = pdp[i]; 977 978 if (topbit) { 979 va = i << (PGSHIFT + level * SEGLENGTH); 980 if (va & __BIT(topbit)) { 981 va |= __BITS(63, topbit); 982 } 983 } 984 if (entry != 0) { 985 paddr_t pa = __SHIFTOUT(entry, PTE_PPN) << PGSHIFT; 986 // check level PPN bits. 987 if (PTE_ISLEAF_P(entry)) { 988 (*pr)("l%u %3zu va 0x%016lx pa 0x%012lx - ", 989 level, i, va, pa); 990 pte_bits(pr, entry); 991 (*pr)("\n"); 992 } else { 993 (*pr)("l%u %3zu va 0x%016lx -> 0x%012lx - ", 994 level, i, va, pa); 995 pte_bits(pr, entry); 996 (*pr)("\n"); 997 if (level == 0) { 998 (*pr)("wtf\n"); 999 continue; 1000 } 1001 if (pte_pde_valid_p(entry)) 1002 dump_ln_table(pa, 0, level - 1, va, pr); 1003 } 1004 } 1005 va += 1UL << (PGSHIFT + level * SEGLENGTH); 1006 } 1007 } 1008 1009 void 1010 pt_dump(void (*pr)(const char *, ...) __printflike(1, 2)) 1011 { 1012 const register_t satp = csr_satp_read(); 1013 size_t topbit = sizeof(long) * NBBY - 1; 1014 1015 #ifdef _LP64 1016 const paddr_t satp_pa = __SHIFTOUT(satp, SATP_PPN) << PGSHIFT; 1017 const uint8_t mode = __SHIFTOUT(satp, SATP_MODE); 1018 u_int level = 1; 1019 1020 switch (mode) { 1021 case SATP_MODE_SV39: 1022 case SATP_MODE_SV48: 1023 topbit = (39 - 1) + (mode - 8) * SEGLENGTH; 1024 level = mode - 6; 1025 break; 1026 } 1027 #endif 1028 (*pr)("topbit = %zu\n", topbit); 1029 1030 (*pr)("satp = 0x%" PRIxREGISTER "\n", satp); 1031 #ifdef _LP64 1032 dump_ln_table(satp_pa, topbit, level, 0, pr); 1033 #endif 1034 } 1035 #endif 1036 1037 void 1038 consinit(void) 1039 { 1040 static bool initialized = false; 1041 const struct fdt_console *cons = fdtbus_get_console(); 1042 const struct fdt_platform *plat = fdt_platform_find(); 1043 1044 if (initialized || cons == NULL) 1045 return; 1046 1047 u_int uart_freq = 0; 1048 extern struct bus_space riscv_generic_bs_tag; 1049 struct fdt_attach_args faa = { 1050 .faa_bst = &riscv_generic_bs_tag, 1051 }; 1052 1053 faa.faa_phandle = fdtbus_get_stdout_phandle(); 1054 if (plat->fp_uart_freq != NULL) 1055 uart_freq = plat->fp_uart_freq(); 1056 1057 cons->consinit(&faa, uart_freq); 1058 1059 initialized = true; 1060 } 1061
Indexes created Sun Oct 25 00:26:45 UTC 2026