1 /* $NetBSD: cpu_subr.c,v 1.69 2026/09/29 05:45:44 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2010, 2019, 2023 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.69 2026/09/29 05:45:44 skrll Exp $"); 34 35 #include "opt_cputype.h" 36 #include "opt_ddb.h" 37 #include "opt_modular.h" 38 #include "opt_multiprocessor.h" 39 40 #include <sys/param.h> 41 #include <sys/atomic.h> 42 #include <sys/bitops.h> 43 #include <sys/cpu.h> 44 #include <sys/device.h> 45 #include <sys/device_impl.h> /* XXX autoconf abuse */ 46 #include <sys/idle.h> 47 #include <sys/intr.h> 48 #include <sys/ipi.h> 49 #include <sys/kernel.h> 50 #include <sys/lwp.h> 51 #include <sys/module.h> 52 #include <sys/paravirt_membar.h> 53 #include <sys/proc.h> 54 #include <sys/ras.h> 55 #include <sys/reboot.h> 56 #include <sys/xcall.h> 57 58 #include <uvm/uvm.h> 59 60 #include <mips/locore.h> 61 #include <mips/regnum.h> 62 #include <mips/pcb.h> 63 #include <mips/cache.h> 64 #include <mips/frame.h> 65 #include <mips/userret.h> 66 #include <mips/pte.h> 67 68 #if defined(DDB) || defined(KGDB) 69 #ifdef DDB 70 #include <mips/db_machdep.h> 71 #include <ddb/db_command.h> 72 #include <ddb/db_output.h> 73 #endif 74 #endif 75 76 #ifdef MIPS64_OCTEON 77 #include <mips/cavium/octeonvar.h> 78 extern struct cpu_softc octeon_cpu_softc[]; 79 #endif 80 81 struct cpu_info cpu_info_store 82 #if defined(MULTIPROCESSOR) && !defined(MIPS64_OCTEON) 83 __section(".data1") 84 __aligned(1LU << ilog2((2*sizeof(struct cpu_info)-1))) 85 #endif 86 = { 87 .ci_curlwp = &lwp0, 88 .ci_tlb_info = &pmap_tlb0_info, 89 .ci_pmap_kern_segtab = &pmap_kern_segtab, 90 .ci_pmap_user_segtab = NULL, 91 #ifdef _LP64 92 .ci_pmap_user_seg0tab = NULL, 93 #endif 94 .ci_cpl = IPL_HIGH, 95 .ci_tlb_slot = -1, 96 #ifdef MULTIPROCESSOR 97 .ci_flags = CPUF_PRIMARY|CPUF_PRESENT|CPUF_RUNNING, 98 #endif 99 #ifdef MIPS64_OCTEON 100 .ci_softc = &octeon_cpu_softc[0], 101 #endif 102 }; 103 104 const pcu_ops_t * const pcu_ops_md_defs[PCU_UNIT_COUNT] = { 105 [PCU_FPU] = &mips_fpu_ops, 106 #if (MIPS32R2 + MIPS64R2) > 0 107 [PCU_DSP] = &mips_dsp_ops, 108 #endif 109 }; 110 111 #ifdef MULTIPROCESSOR 112 struct cpu_info * cpuid_infos[MAXCPUS] = { 113 [0] = &cpu_info_store, 114 }; 115 116 kcpuset_t *cpus_halted; 117 kcpuset_t *cpus_hatched; 118 kcpuset_t *cpus_paused; 119 kcpuset_t *cpus_resumed; 120 kcpuset_t *cpus_running; 121 122 static void cpu_ipi_wait(const char *, const kcpuset_t *, const kcpuset_t *); 123 124 struct cpu_info * 125 cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id, 126 cpuid_t cpu_core_id, cpuid_t cpu_smt_id) 127 { 128 129 KASSERT(cpu_id < MAXCPUS); 130 131 #ifdef MIPS64_OCTEON 132 const int exc_step = 1 << MIPS_EBASE_EXC_BASE_SHIFT; 133 vaddr_t exc_page = MIPS_UTLB_MISS_EXC_VEC + exc_step * cpu_id; 134 __CTASSERT(sizeof(struct cpu_info) + sizeof(struct pmap_tlb_info) 135 <= exc_step - 0x280); 136 137 struct cpu_info * const ci = ((struct cpu_info *)(exc_page + exc_step)) - 1; 138 memset((void *)exc_page, 0, exc_step); 139 140 if (ti == NULL) { 141 ti = ((struct pmap_tlb_info *)ci) - 1; 142 pmap_tlb_info_init(ti); 143 } 144 #else 145 const vaddr_t cpu_info_offset = (vaddr_t)&cpu_info_store & PAGE_MASK; 146 struct pglist pglist; 147 int error; 148 149 /* 150 * Grab a page from the first 512MB (mappable by KSEG0) to use to store 151 * exception vectors and cpu_info for this cpu. 152 */ 153 error = uvm_pglistalloc(PAGE_SIZE, 154 0, MIPS_KSEG1_START - MIPS_KSEG0_START, 155 PAGE_SIZE, PAGE_SIZE, &pglist, 1, false); 156 if (error) 157 return NULL; 158 159 const paddr_t pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 160 const vaddr_t va = MIPS_PHYS_TO_KSEG0(pa); 161 struct cpu_info * const ci = (void *) (va + cpu_info_offset); 162 memset((void *)va, 0, PAGE_SIZE); 163 164 /* 165 * If we weren't passed a pmap_tlb_info to use, the caller wants us 166 * to take care of that for him. Since we have room left over in the 167 * page we just allocated, just use a piece of that for it. 168 */ 169 if (ti == NULL) { 170 if (cpu_info_offset >= sizeof(*ti)) { 171 ti = (void *) va; 172 } else { 173 KASSERT(PAGE_SIZE - cpu_info_offset + sizeof(*ci) >= sizeof(*ti)); 174 ti = (struct pmap_tlb_info *)(va + PAGE_SIZE) - 1; 175 } 176 pmap_tlb_info_init(ti); 177 } 178 179 /* 180 * Attach its TLB info (which must be direct-mapped) 181 */ 182 #ifdef _LP64 183 KASSERT(MIPS_KSEG0_P(ti) || MIPS_XKPHYS_P(ti)); 184 #else 185 KASSERT(MIPS_KSEG0_P(ti)); 186 #endif 187 #endif /* MIPS64_OCTEON */ 188 189 KASSERT(cpu_id != 0); 190 ci->ci_cpuid = cpu_id; 191 ci->ci_pmap_kern_segtab = &pmap_kern_segtab, 192 ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq; 193 ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq; 194 ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz; 195 ci->ci_divisor_delay = cpu_info_store.ci_divisor_delay; 196 ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip; 197 ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count; 198 199 cpu_topology_set(ci, cpu_package_id, cpu_core_id, cpu_smt_id, 0); 200 201 pmap_md_alloc_ephemeral_address_space(ci); 202 203 mi_cpu_attach(ci); 204 205 pmap_tlb_info_attach(ti, ci); 206 207 return ci; 208 } 209 #endif /* MULTIPROCESSOR */ 210 211 static void 212 cpu_hwrena_setup(void) 213 { 214 #if (MIPS32R2 + MIPS64R2) > 0 215 const int cp0flags = mips_options.mips_cpu->cpu_cp0flags; 216 217 if ((cp0flags & MIPS_CP0FL_USE) == 0) 218 return; 219 220 if (CPUISMIPSNNR2) { 221 mipsNN_cp0_hwrena_write( 222 (MIPS_HAS_USERLOCAL ? MIPS_HWRENA_ULR : 0) 223 | MIPS_HWRENA_CCRES 224 | MIPS_HWRENA_CC 225 | MIPS_HWRENA_SYNCI_STEP 226 | MIPS_HWRENA_CPUNUM); 227 if (MIPS_HAS_USERLOCAL) { 228 mipsNN_cp0_userlocal_write(curlwp->l_private); 229 } 230 } 231 #endif 232 } 233 234 void 235 cpu_attach_common(device_t self, struct cpu_info *ci) 236 { 237 const char * const xname = device_xname(self); 238 239 /* 240 * Cross link cpu_info and its device together 241 * 242 * XXX autoconf abuse: Can't use device_set_private here 243 * because some callers already do so -- and some callers 244 * (sbmips cpu_attach) already have a softc allocated by 245 * autoconf. 246 */ 247 ci->ci_dev = self; 248 self->dv_private = ci; 249 KASSERT(ci->ci_idepth == 0); 250 251 evcnt_attach_dynamic(&ci->ci_ev_count_compare, 252 EVCNT_TYPE_INTR, NULL, xname, 253 "int 5 (clock)"); 254 evcnt_attach_dynamic(&ci->ci_ev_count_compare_missed, 255 EVCNT_TYPE_INTR, NULL, xname, 256 "int 5 (clock) missed"); 257 evcnt_attach_dynamic(&ci->ci_ev_fpu_loads, 258 EVCNT_TYPE_MISC, NULL, xname, 259 "fpu loads"); 260 evcnt_attach_dynamic(&ci->ci_ev_fpu_saves, 261 EVCNT_TYPE_MISC, NULL, xname, 262 "fpu saves"); 263 evcnt_attach_dynamic(&ci->ci_ev_dsp_loads, 264 EVCNT_TYPE_MISC, NULL, xname, 265 "dsp loads"); 266 evcnt_attach_dynamic(&ci->ci_ev_dsp_saves, 267 EVCNT_TYPE_MISC, NULL, xname, 268 "dsp saves"); 269 evcnt_attach_dynamic(&ci->ci_ev_tlbmisses, 270 EVCNT_TYPE_TRAP, NULL, xname, 271 "tlb misses"); 272 273 #ifdef MULTIPROCESSOR 274 if (ci != &cpu_info_store) { 275 /* 276 * Tail insert this onto the list of cpu_info's. 277 * atomic_store_release matches PTR_L/SYNC_ACQ in 278 * locore_octeon.S (XXX what about non-Octeon?). 279 */ 280 KASSERT(cpuid_infos[ci->ci_cpuid] == NULL); 281 atomic_store_release(&cpuid_infos[ci->ci_cpuid], ci); 282 membar_producer(); /* Cavium sync plunger */ 283 284 /* 285 * The BP has its ci_shootdowncpus cpuset created in 286 * pmap_bootstrap. 287 */ 288 kcpuset_create(&ci->ci_shootdowncpus, true); 289 } 290 KASSERT(cpuid_infos[ci->ci_cpuid] != NULL); 291 evcnt_attach_dynamic(&ci->ci_evcnt_synci_activate_rqst, 292 EVCNT_TYPE_MISC, NULL, xname, 293 "syncicache activate request"); 294 evcnt_attach_dynamic(&ci->ci_evcnt_synci_deferred_rqst, 295 EVCNT_TYPE_MISC, NULL, xname, 296 "syncicache deferred request"); 297 evcnt_attach_dynamic(&ci->ci_evcnt_synci_ipi_rqst, 298 EVCNT_TYPE_MISC, NULL, xname, 299 "syncicache ipi request"); 300 evcnt_attach_dynamic(&ci->ci_evcnt_synci_onproc_rqst, 301 EVCNT_TYPE_MISC, NULL, xname, 302 "syncicache onproc request"); 303 304 /* 305 * Initialize IPI framework for this cpu instance 306 */ 307 ipi_init(ci); 308 309 kcpuset_create(&ci->ci_multicastcpus, true); 310 kcpuset_create(&ci->ci_watchcpus, true); 311 kcpuset_create(&ci->ci_ddbcpus, true); 312 #endif 313 } 314 315 void 316 cpu_startup_common(void) 317 { 318 vaddr_t minaddr, maxaddr; 319 char pbuf[9]; /* "99999 MB" */ 320 321 pmap_tlb_info_evcnt_attach(&pmap_tlb0_info); 322 323 #ifdef MULTIPROCESSOR 324 kcpuset_create(&cpus_halted, true); 325 kcpuset_create(&cpus_hatched, true); 326 kcpuset_create(&cpus_paused, true); 327 kcpuset_create(&cpus_resumed, true); 328 kcpuset_create(&cpus_running, true); 329 KASSERT(cpus_halted != NULL); 330 KASSERT(cpus_hatched != NULL); 331 KASSERT(cpus_paused != NULL); 332 KASSERT(cpus_resumed != NULL); 333 KASSERT(cpus_running != NULL); 334 kcpuset_set(cpus_hatched, cpu_number()); 335 kcpuset_set(cpus_running, cpu_number()); 336 #endif 337 338 cpu_hwrena_setup(); 339 340 /* 341 * Good {morning,afternoon,evening,night}. 342 */ 343 printf("%s%s", copyright, version); 344 printf("%s\n", cpu_getmodel()); 345 format_bytes(pbuf, sizeof(pbuf), ctob(physmem)); 346 printf("total memory = %s\n", pbuf); 347 348 minaddr = 0; 349 /* 350 * Allocate a submap for physio. 351 */ 352 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 353 VM_PHYS_SIZE, 0, FALSE, NULL); 354 355 /* 356 * (No need to allocate an mbuf cluster submap. Mbuf clusters 357 * are allocated via the pool allocator, and we use KSEG/XKPHYS to 358 * map those pages.) 359 */ 360 361 format_bytes(pbuf, sizeof(pbuf), ptoa(uvm_availmem(false))); 362 printf("avail memory = %s\n", pbuf); 363 364 #if defined(__mips_n32) 365 module_machine = "mips-n32"; 366 #endif 367 } 368 369 void 370 cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags) 371 { 372 const struct trapframe *tf = l->l_md.md_utf; 373 __greg_t *gr = mcp->__gregs; 374 __greg_t ras_pc; 375 376 /* Save register context. Dont copy R0 - it is always 0 */ 377 memcpy(&gr[_REG_AT], &tf->tf_regs[_R_AST], sizeof(mips_reg_t) * 31); 378 379 gr[_REG_MDLO] = tf->tf_regs[_R_MULLO]; 380 gr[_REG_MDHI] = tf->tf_regs[_R_MULHI]; 381 gr[_REG_CAUSE] = tf->tf_regs[_R_CAUSE]; 382 gr[_REG_EPC] = tf->tf_regs[_R_PC]; 383 gr[_REG_SR] = tf->tf_regs[_R_SR]; 384 mcp->_mc_tlsbase = (intptr_t)l->l_private; 385 386 if ((ras_pc = (intptr_t)ras_lookup(l->l_proc, 387 (void *) (intptr_t)gr[_REG_EPC])) != -1) 388 gr[_REG_EPC] = ras_pc; 389 390 *flags |= _UC_CPU | _UC_TLSBASE; 391 392 /* Save floating point register context, if any. */ 393 KASSERT(l == curlwp); 394 if (fpu_used_p(l)) { 395 size_t fplen; 396 /* 397 * If this process is the current FP owner, dump its 398 * context to the PCB first. 399 */ 400 fpu_save(l); 401 402 /* 403 * The PCB FP regs struct includes the FP CSR, so use the 404 * size of __fpregs.__fp_r when copying. 405 */ 406 #if !defined(__mips_o32) 407 if (_MIPS_SIM_NEWABI_P(l->l_proc->p_md.md_abi)) { 408 #endif 409 fplen = sizeof(struct fpreg); 410 #if !defined(__mips_o32) 411 } else { 412 fplen = sizeof(struct fpreg_oabi); 413 } 414 #endif 415 struct pcb * const pcb = lwp_getpcb(l); 416 memcpy(&mcp->__fpregs, &pcb->pcb_fpregs, fplen); 417 *flags |= _UC_FPU; 418 } 419 } 420 421 int 422 cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp) 423 { 424 425 /* XXX: Do we validate the addresses?? */ 426 return 0; 427 } 428 429 int 430 cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags) 431 { 432 struct trapframe *tf = l->l_md.md_utf; 433 struct proc *p = l->l_proc; 434 const __greg_t *gr = mcp->__gregs; 435 int error; 436 437 /* Restore register context, if any. */ 438 if (flags & _UC_CPU) { 439 error = cpu_mcontext_validate(l, mcp); 440 if (error) 441 return error; 442 443 /* Save register context. */ 444 445 #ifdef __mips_n32 446 CTASSERT(_R_AST == _REG_AT); 447 if (__predict_false(p->p_md.md_abi == _MIPS_BSD_API_O32)) { 448 const mcontext_o32_t *mcp32 = (const mcontext_o32_t *)mcp; 449 const __greg32_t *gr32 = mcp32->__gregs; 450 for (size_t i = _R_AST; i < 32; i++) { 451 tf->tf_regs[i] = gr32[i]; 452 } 453 } else 454 #endif 455 memcpy(&tf->tf_regs[_R_AST], &gr[_REG_AT], 456 sizeof(mips_reg_t) * 31); 457 458 tf->tf_regs[_R_MULLO] = gr[_REG_MDLO]; 459 tf->tf_regs[_R_MULHI] = gr[_REG_MDHI]; 460 tf->tf_regs[_R_CAUSE] = gr[_REG_CAUSE]; 461 tf->tf_regs[_R_PC] = gr[_REG_EPC]; 462 /* Do not restore SR. */ 463 } 464 465 /* Restore the private thread context */ 466 if (flags & _UC_TLSBASE) { 467 lwp_setprivate(l, (void *)(intptr_t)mcp->_mc_tlsbase); 468 } 469 470 /* Restore floating point register context, if any. */ 471 if (flags & _UC_FPU) { 472 size_t fplen; 473 474 /* Disable the FPU contents. */ 475 fpu_discard(l); 476 477 #if !defined(__mips_o32) 478 if (_MIPS_SIM_NEWABI_P(l->l_proc->p_md.md_abi)) { 479 #endif 480 fplen = sizeof(struct fpreg); 481 #if !defined(__mips_o32) 482 } else { 483 fplen = sizeof(struct fpreg_oabi); 484 } 485 #endif 486 /* 487 * The PCB FP regs struct includes the FP CSR, so use the 488 * proper size of fpreg when copying. 489 */ 490 struct pcb * const pcb = lwp_getpcb(l); 491 memcpy(&pcb->pcb_fpregs, &mcp->__fpregs, fplen); 492 } 493 494 mutex_enter(p->p_lock); 495 if (flags & _UC_SETSTACK) 496 l->l_sigstk.ss_flags |= SS_ONSTACK; 497 if (flags & _UC_CLRSTACK) 498 l->l_sigstk.ss_flags &= ~SS_ONSTACK; 499 mutex_exit(p->p_lock); 500 501 return (0); 502 } 503 504 void 505 cpu_need_resched(struct cpu_info *ci, struct lwp *l, int flags) 506 { 507 508 KASSERT(kpreempt_disabled()); 509 510 if ((flags & RESCHED_KPREEMPT) != 0) { 511 #ifdef __HAVE_PREEMPTION 512 if ((flags & RESCHED_REMOTE) != 0) { 513 cpu_send_ipi(ci, IPI_KPREEMPT); 514 } else { 515 softint_trigger(SOFTINT_KPREEMPT); 516 } 517 #endif 518 return; 519 } 520 if ((flags & RESCHED_REMOTE) != 0) { 521 #ifdef MULTIPROCESSOR 522 cpu_send_ipi(ci, IPI_AST); 523 #endif 524 } else { 525 l->l_md.md_astpending = 1; /* force call to ast() */ 526 } 527 } 528 529 uint32_t 530 cpu_clkf_usermode_mask(void) 531 { 532 533 return CPUISMIPS3 ? MIPS_SR_KSU_USER : MIPS_SR_KU_PREV; 534 } 535 536 void 537 cpu_signotify(struct lwp *l) 538 { 539 540 KASSERT(kpreempt_disabled()); 541 #ifdef __HAVE_FAST_SOFTINTS 542 KASSERT(lwp_locked(l, NULL)); 543 #endif 544 545 if (l->l_cpu != curcpu()) { 546 #ifdef MULTIPROCESSOR 547 cpu_send_ipi(l->l_cpu, IPI_AST); 548 #endif 549 } else { 550 l->l_md.md_astpending = 1; /* force call to ast() */ 551 } 552 } 553 554 void 555 cpu_need_proftick(struct lwp *l) 556 { 557 558 KASSERT(kpreempt_disabled()); 559 KASSERT(l->l_cpu == curcpu()); 560 561 l->l_pflag |= LP_OWEUPC; 562 l->l_md.md_astpending = 1; /* force call to ast() */ 563 } 564 565 #ifdef __HAVE_PREEMPTION 566 bool 567 cpu_kpreempt_enter(uintptr_t where, int s) 568 { 569 570 KASSERT(kpreempt_disabled()); 571 572 #if 0 573 if (where == (intptr_t)-2) { 574 KASSERT(curcpu()->ci_mtx_count == 0); 575 /* 576 * We must be called via kern_intr (which already checks for 577 * IPL_NONE so of course we call be preempted). 578 */ 579 return true; 580 } 581 /* 582 * We are called from KPREEMPT_ENABLE(). If we are at IPL_NONE, 583 * of course we can be preempted. If we aren't, ask for a 584 * softint so that kern_intr can call kpreempt. 585 */ 586 if (s == IPL_NONE) { 587 KASSERT(curcpu()->ci_mtx_count == 0); 588 return true; 589 } 590 softint_trigger(SOFTINT_KPREEMPT); 591 #endif 592 return false; 593 } 594 595 void 596 cpu_kpreempt_exit(uintptr_t where) 597 { 598 599 /* do nothing */ 600 } 601 602 /* 603 * Return true if preemption is disabled for MD reasons. Must be called 604 * with preemption disabled, and thus is only for diagnostic checks. 605 */ 606 bool 607 cpu_kpreempt_disabled(void) 608 { 609 610 /* 611 * Any elevated IPL disables preemption. 612 */ 613 return curcpu()->ci_cpl > IPL_NONE; 614 } 615 #endif /* __HAVE_PREEMPTION */ 616 617 void 618 cpu_idle(void) 619 { 620 void (*const mach_idle)(void) = mips_locoresw.lsw_cpu_idle; 621 struct cpu_info * const ci = curcpu(); 622 623 while (!ci->ci_want_resched) { 624 #ifdef __HAVE_FAST_SOFTINTS 625 KASSERT(ci->ci_data.cpu_softints == 0); 626 #endif 627 (*mach_idle)(); 628 } 629 } 630 631 bool 632 cpu_intr_p(void) 633 { 634 int idepth; 635 long pctr; 636 lwp_t *l; 637 638 l = curlwp; 639 do { 640 pctr = lwp_pctr(); 641 idepth = l->l_cpu->ci_idepth; 642 } while (__predict_false(pctr != lwp_pctr())); 643 644 return idepth != 0; 645 } 646 647 #ifdef MULTIPROCESSOR 648 649 void 650 cpu_broadcast_ipi(int tag) 651 { 652 653 // No reason to remove ourselves since multicast_ipi will do that for us 654 cpu_multicast_ipi(cpus_running, tag); 655 } 656 657 void 658 cpu_multicast_ipi(const kcpuset_t *kcp, int tag) 659 { 660 struct cpu_info * const ci = curcpu(); 661 kcpuset_t *kcp2 = ci->ci_multicastcpus; 662 663 if (kcpuset_match(cpus_running, ci->ci_kcpuset)) 664 return; 665 666 kcpuset_copy(kcp2, kcp); 667 kcpuset_remove(kcp2, ci->ci_kcpuset); 668 for (cpuid_t cii; (cii = kcpuset_ffs(kcp2)) != 0; ) { 669 kcpuset_clear(kcp2, --cii); 670 (void)cpu_send_ipi(cpu_lookup(cii), tag); 671 } 672 } 673 674 int 675 cpu_send_ipi(struct cpu_info *ci, int tag) 676 { 677 678 return (*mips_locoresw.lsw_send_ipi)(ci, tag); 679 } 680 681 static void 682 cpu_ipi_wait(const char *s, const kcpuset_t *watchset, const kcpuset_t *wanted) 683 { 684 bool done = false; 685 struct cpu_info * const ci = curcpu(); 686 kcpuset_t *kcp = ci->ci_watchcpus; 687 688 /* some finite amount of time */ 689 690 for (u_long limit = curcpu()->ci_cpu_freq/10; !done && limit--; ) { 691 kcpuset_copy(kcp, watchset); 692 kcpuset_intersect(kcp, wanted); 693 done = kcpuset_match(kcp, wanted); 694 } 695 696 if (!done) { 697 cpuid_t cii; 698 kcpuset_copy(kcp, wanted); 699 kcpuset_remove(kcp, watchset); 700 if ((cii = kcpuset_ffs(kcp)) != 0) { 701 printf("Failed to %s:", s); 702 do { 703 kcpuset_clear(kcp, --cii); 704 printf(" cpu%lu", cii); 705 } while ((cii = kcpuset_ffs(kcp)) != 0); 706 printf("\n"); 707 } 708 } 709 } 710 711 /* 712 * Halt this cpu 713 */ 714 void 715 cpu_halt(void) 716 { 717 cpuid_t cii = cpu_index(curcpu()); 718 719 printf("cpu%lu: shutting down\n", cii); 720 kcpuset_atomic_set(cpus_halted, cii); 721 spl0(); /* allow interrupts e.g. further ipi ? */ 722 for (;;) ; /* spin */ 723 724 /* NOTREACHED */ 725 } 726 727 /* 728 * Halt all running cpus, excluding current cpu. 729 */ 730 void 731 cpu_halt_others(void) 732 { 733 kcpuset_t *kcp; 734 735 // If we are the only CPU running, there's nothing to do. 736 if (kcpuset_match(cpus_running, curcpu()->ci_kcpuset)) 737 return; 738 739 // Get all running CPUs 740 kcpuset_clone(&kcp, cpus_running); 741 // Remove ourself 742 kcpuset_remove(kcp, curcpu()->ci_kcpuset); 743 // Remove any halted CPUs 744 kcpuset_remove(kcp, cpus_halted); 745 // If there are CPUs left, send the IPIs 746 if (!kcpuset_iszero(kcp)) { 747 cpu_multicast_ipi(kcp, IPI_HALT); 748 cpu_ipi_wait("halt", cpus_halted, kcp); 749 } 750 kcpuset_destroy(kcp); 751 752 /* 753 * TBD 754 * Depending on available firmware methods, other cpus will 755 * either shut down themselves, or spin and wait for us to 756 * stop them. 757 */ 758 } 759 760 /* 761 * Pause this cpu 762 */ 763 void 764 cpu_pause(struct reg *regsp) 765 { 766 int s = splhigh(); 767 cpuid_t cii = cpu_index(curcpu()); 768 769 if (__predict_false(cold)) { 770 splx(s); 771 return; 772 } 773 774 do { 775 kcpuset_atomic_set(cpus_paused, cii); 776 do { 777 ; 778 } while (kcpuset_isset(cpus_paused, cii)); 779 kcpuset_atomic_set(cpus_resumed, cii); 780 #if defined(DDB) 781 if (ddb_running_on_this_cpu_p()) 782 cpu_Debugger(); 783 if (ddb_running_on_any_cpu_p()) 784 continue; 785 #endif 786 } while (false); 787 788 splx(s); 789 } 790 791 /* 792 * Pause all running cpus, excluding current cpu. 793 */ 794 void 795 cpu_pause_others(void) 796 { 797 struct cpu_info * const ci = curcpu(); 798 799 if (cold || kcpuset_match(cpus_running, ci->ci_kcpuset)) 800 return; 801 802 kcpuset_t *kcp = ci->ci_ddbcpus; 803 804 kcpuset_copy(kcp, cpus_running); 805 kcpuset_remove(kcp, ci->ci_kcpuset); 806 kcpuset_remove(kcp, cpus_paused); 807 808 cpu_broadcast_ipi(IPI_SUSPEND); 809 cpu_ipi_wait("pause", cpus_paused, kcp); 810 } 811 812 /* 813 * Resume a single cpu 814 */ 815 void 816 cpu_resume(cpuid_t cii) 817 { 818 819 if (__predict_false(cold)) 820 return; 821 822 struct cpu_info * const ci = curcpu(); 823 kcpuset_t *kcp = ci->ci_ddbcpus; 824 825 kcpuset_set(kcp, cii); 826 kcpuset_atomicly_remove(cpus_resumed, cpus_resumed); 827 kcpuset_atomic_clear(cpus_paused, cii); 828 829 cpu_ipi_wait("resume", cpus_resumed, kcp); 830 } 831 832 /* 833 * Resume all paused cpus. 834 */ 835 void 836 cpu_resume_others(void) 837 { 838 839 if (__predict_false(cold)) 840 return; 841 842 struct cpu_info * const ci = curcpu(); 843 kcpuset_t *kcp = ci->ci_ddbcpus; 844 845 kcpuset_atomicly_remove(cpus_resumed, cpus_resumed); 846 kcpuset_copy(kcp, cpus_paused); 847 kcpuset_atomicly_remove(cpus_paused, cpus_paused); 848 849 /* CPUs awake on cpus_paused clear */ 850 cpu_ipi_wait("resume", cpus_resumed, kcp); 851 } 852 853 bool 854 cpu_is_paused(cpuid_t cii) 855 { 856 857 return !cold && kcpuset_isset(cpus_paused, cii); 858 } 859 860 #ifdef DDB 861 void 862 cpu_debug_dump(void) 863 { 864 CPU_INFO_ITERATOR cii; 865 struct cpu_info *ci; 866 char running, hatched, paused, resumed, halted; 867 db_printf("CPU CPUID STATE CPUINFO CPL INT MTX IPIS(A/R)\n"); 868 for (CPU_INFO_FOREACH(cii, ci)) { 869 hatched = (kcpuset_isset(cpus_hatched, cpu_index(ci)) ? 'H' : '-'); 870 running = (kcpuset_isset(cpus_running, cpu_index(ci)) ? 'R' : '-'); 871 paused = (kcpuset_isset(cpus_paused, cpu_index(ci)) ? 'P' : '-'); 872 resumed = (kcpuset_isset(cpus_resumed, cpu_index(ci)) ? 'r' : '-'); 873 halted = (kcpuset_isset(cpus_halted, cpu_index(ci)) ? 'h' : '-'); 874 db_printf("%3d 0x%03lx %c%c%c%c%c %p " 875 "%3d %3d %3d " 876 "0x%02" PRIx64 "/0x%02" PRIx64 "\n", 877 cpu_index(ci), ci->ci_cpuid, 878 running, hatched, paused, resumed, halted, 879 ci, ci->ci_cpl, ci->ci_idepth, ci->ci_mtx_count, 880 ci->ci_active_ipis, ci->ci_request_ipis); 881 } 882 } 883 #endif 884 885 void 886 cpu_hatch(struct cpu_info *ci) 887 { 888 struct pmap_tlb_info * const ti = cpu_tlb_info(ci); 889 890 /* 891 * Invalidate all the TLB enties (even wired ones) and then reserve 892 * space for the wired TLB entries. 893 */ 894 mips3_cp0_wired_write(0); 895 tlb_invalidate_all(); 896 mips3_cp0_wired_write(ti->ti_wired); 897 898 /* 899 * Setup HWRENA and USERLOCAL COP0 registers (MIPSxxR2). 900 */ 901 cpu_hwrena_setup(); 902 903 /* 904 * If we are using register zero relative addressing to access cpu_info 905 * in the exception vectors, enter that mapping into TLB now. 906 */ 907 if (ci->ci_tlb_slot >= 0) { 908 const uint32_t tlb_lo = MIPS3_PG_G|MIPS3_PG_V 909 | mips3_paddr_to_tlbpfn((vaddr_t)ci); 910 const struct tlbmask tlbmask = { 911 .tlb_hi = -PAGE_SIZE | KERNEL_PID, 912 #if (PGSHIFT & 1) 913 .tlb_lo0 = tlb_lo, 914 .tlb_lo1 = tlb_lo + MIPS3_PG_NEXT, 915 #else 916 .tlb_lo0 = 0, 917 .tlb_lo1 = tlb_lo, 918 #endif 919 .tlb_mask = -1, 920 }; 921 922 tlb_invalidate_addr(tlbmask.tlb_hi, KERNEL_PID); 923 tlb_write_entry(ci->ci_tlb_slot, &tlbmask); 924 } 925 926 /* 927 * Flush the icache just be sure. 928 */ 929 mips_icache_sync_all(); 930 931 /* 932 * Let this CPU do its own initialization (for things that have to be 933 * done on the local CPU). 934 */ 935 (*mips_locoresw.lsw_cpu_init)(ci); 936 937 // Show this CPU as present. 938 atomic_or_ulong(&ci->ci_flags, CPUF_PRESENT); 939 940 /* 941 * Announce we are hatched 942 */ 943 kcpuset_atomic_set(cpus_hatched, cpu_index(ci)); 944 945 /* 946 * Now wait to be set free! 947 */ 948 while (! kcpuset_isset(cpus_running, cpu_index(ci))) { 949 /* spin, spin, spin */ 950 } 951 952 /* 953 * initialize the MIPS count/compare clock 954 */ 955 mips3_cp0_count_write(ci->ci_data.cpu_cc_skew); 956 KASSERT(ci->ci_cycles_per_hz != 0); 957 ci->ci_next_cp0_clk_intr = ci->ci_data.cpu_cc_skew + ci->ci_cycles_per_hz; 958 mips3_cp0_compare_write(ci->ci_next_cp0_clk_intr); 959 ci->ci_data.cpu_cc_skew = 0; 960 961 /* 962 * Let this CPU do its own post-running initialization 963 * (for things that have to be done on the local CPU). 964 */ 965 (*mips_locoresw.lsw_cpu_run)(ci); 966 967 /* 968 * Now turn on interrupts (and verify they are on). 969 */ 970 spl0(); 971 KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl); 972 KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 973 974 kcpuset_atomic_set(pmap_kernel()->pm_onproc, cpu_index(ci)); 975 kcpuset_atomic_set(pmap_kernel()->pm_active, cpu_index(ci)); 976 977 /* 978 * And do a tail call to idle_loop 979 */ 980 idle_loop(NULL); 981 } 982 983 void 984 cpu_boot_secondary_processors(void) 985 { 986 CPU_INFO_ITERATOR cii; 987 struct cpu_info *ci; 988 989 if ((boothowto & RB_MD1) != 0) 990 return; 991 992 for (CPU_INFO_FOREACH(cii, ci)) { 993 if (CPU_IS_PRIMARY(ci)) 994 continue; 995 KASSERT(ci->ci_data.cpu_idlelwp); 996 997 /* 998 * Skip this CPU if it didn't successfully hatch. 999 */ 1000 if (!kcpuset_isset(cpus_hatched, cpu_index(ci))) 1001 continue; 1002 1003 ci->ci_data.cpu_cc_skew = mips3_cp0_count_read(); 1004 atomic_or_ulong(&ci->ci_flags, CPUF_RUNNING); 1005 kcpuset_set(cpus_running, cpu_index(ci)); 1006 // Spin until the cpu calls idle_loop 1007 for (u_int i = 0; i < 10000; i++) { 1008 if (kcpuset_isset(kcpuset_running, cpu_index(ci))) 1009 break; 1010 delay(1000); 1011 } 1012 } 1013 } 1014 1015 void 1016 xc_send_ipi(struct cpu_info *ci) 1017 { 1018 1019 (*mips_locoresw.lsw_send_ipi)(ci, IPI_XCALL); 1020 } 1021 1022 void 1023 cpu_ipi(struct cpu_info *ci) 1024 { 1025 1026 (*mips_locoresw.lsw_send_ipi)(ci, IPI_GENERIC); 1027 } 1028 1029 #endif /* MULTIPROCESSOR */ 1030 1031 void 1032 cpu_offline_md(void) 1033 { 1034 1035 (*mips_locoresw.lsw_cpu_offline_md)(); 1036 } 1037 1038 #ifdef _LP64 1039 void 1040 cpu_vmspace_exec(lwp_t *l, vaddr_t start, vaddr_t end) 1041 { 1042 /* 1043 * We need to turn on/off UX so that copyout/copyin will work 1044 * well before setreg gets called. 1045 */ 1046 uint32_t sr = mips_cp0_status_read(); 1047 1048 if (end != (uint32_t) end) { 1049 mips_cp0_status_write(sr | MIPS3_SR_UX); 1050 } else { 1051 mips_cp0_status_write(sr & ~MIPS3_SR_UX); 1052 } 1053 } 1054 #endif 1055 1056 int 1057 cpu_lwp_setprivate(lwp_t *l, void *v) 1058 { 1059 1060 #if (MIPS32R2 + MIPS64R2) > 0 1061 if (l == curlwp && MIPS_HAS_USERLOCAL) { 1062 mipsNN_cp0_userlocal_write(v); 1063 } 1064 #endif 1065 return 0; 1066 } 1067 1068 1069 #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 1070 1071 #if (CPUWATCH_MAX != 8) 1072 # error CPUWATCH_MAX 1073 #endif 1074 1075 /* 1076 * cpuwatch_discover - determine how many COP0 watchpoints this CPU supports 1077 */ 1078 u_int 1079 cpuwatch_discover(void) 1080 { 1081 int i; 1082 1083 for (i=0; i < CPUWATCH_MAX; i++) { 1084 uint32_t watchhi = mipsNN_cp0_watchhi_read(i); 1085 if ((watchhi & __BIT(31)) == 0) /* test 'M' bit */ 1086 break; 1087 } 1088 return i + 1; 1089 } 1090 1091 void 1092 cpuwatch_free(cpu_watchpoint_t *cwp) 1093 { 1094 #ifdef DIAGNOSTIC 1095 struct cpu_info * const ci = curcpu(); 1096 1097 KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && 1098 cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); 1099 #endif 1100 cwp->cw_mode = 0; 1101 cwp->cw_asid = 0; 1102 cwp->cw_addr = 0; 1103 cpuwatch_clr(cwp); 1104 } 1105 1106 /* 1107 * cpuwatch_alloc 1108 * find an empty slot 1109 * no locking for the table since it is CPU private 1110 */ 1111 cpu_watchpoint_t * 1112 cpuwatch_alloc(void) 1113 { 1114 struct cpu_info * const ci = curcpu(); 1115 cpu_watchpoint_t *cwp; 1116 1117 for (int i=0; i < ci->ci_cpuwatch_count; i++) { 1118 cwp = &ci->ci_cpuwatch_tab[i]; 1119 if ((cwp->cw_mode & CPUWATCH_RWX) == 0) 1120 return cwp; 1121 } 1122 return NULL; 1123 } 1124 1125 1126 void 1127 cpuwatch_set_all(void) 1128 { 1129 struct cpu_info * const ci = curcpu(); 1130 cpu_watchpoint_t *cwp; 1131 int i; 1132 1133 for (i=0; i < ci->ci_cpuwatch_count; i++) { 1134 cwp = &ci->ci_cpuwatch_tab[i]; 1135 if ((cwp->cw_mode & CPUWATCH_RWX) != 0) 1136 cpuwatch_set(cwp); 1137 } 1138 } 1139 1140 void 1141 cpuwatch_clr_all(void) 1142 { 1143 struct cpu_info * const ci = curcpu(); 1144 cpu_watchpoint_t *cwp; 1145 int i; 1146 1147 for (i=0; i < ci->ci_cpuwatch_count; i++) { 1148 cwp = &ci->ci_cpuwatch_tab[i]; 1149 if ((cwp->cw_mode & CPUWATCH_RWX) != 0) 1150 cpuwatch_clr(cwp); 1151 } 1152 } 1153 1154 /* 1155 * cpuwatch_set - establish a MIPS COP0 watchpoint 1156 */ 1157 void 1158 cpuwatch_set(cpu_watchpoint_t *cwp) 1159 { 1160 struct cpu_info * const ci = curcpu(); 1161 uint32_t watchhi; 1162 register_t watchlo; 1163 int cwnum = cwp - &ci->ci_cpuwatch_tab[0]; 1164 1165 KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && 1166 cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); 1167 1168 watchlo = cwp->cw_addr; 1169 if (cwp->cw_mode & CPUWATCH_WRITE) 1170 watchlo |= __BIT(0); 1171 if (cwp->cw_mode & CPUWATCH_READ) 1172 watchlo |= __BIT(1); 1173 if (cwp->cw_mode & CPUWATCH_EXEC) 1174 watchlo |= __BIT(2); 1175 1176 if (cwp->cw_mode & CPUWATCH_ASID) 1177 watchhi = cwp->cw_asid << 16; /* addr qualified by asid */ 1178 else 1179 watchhi = __BIT(30); /* addr not qual. by asid (Global) */ 1180 if (cwp->cw_mode & CPUWATCH_MASK) 1181 watchhi |= cwp->cw_mask; /* set "dont care" addr match bits */ 1182 1183 mipsNN_cp0_watchhi_write(cwnum, watchhi); 1184 mipsNN_cp0_watchlo_write(cwnum, watchlo); 1185 } 1186 1187 /* 1188 * cpuwatch_clr - disestablish a MIPS COP0 watchpoint 1189 */ 1190 void 1191 cpuwatch_clr(cpu_watchpoint_t *cwp) 1192 { 1193 struct cpu_info * const ci = curcpu(); 1194 int cwnum = cwp - &ci->ci_cpuwatch_tab[0]; 1195 1196 KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] && 1197 cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]); 1198 1199 mipsNN_cp0_watchhi_write(cwnum, 0); 1200 mipsNN_cp0_watchlo_write(cwnum, 0); 1201 } 1202 1203 #endif /* (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 */ 1204 1205 #if (MIPS2 + MIPS3 + MIPS4 + MIPS5 + MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 1206 void 1207 paravirt_membar_sync(void) 1208 { 1209 1210 /* 1211 * Store-before-load ordering with respect to matching logic 1212 * on the hypervisor side. 1213 * 1214 * This is the same as membar_sync, but guaranteed never to be 1215 * conditionalized or hotpatched away even on uniprocessor 1216 * builds and boots -- because under virtualization, we still 1217 * have to coordinate with a `device' backed by a hypervisor 1218 * that is potentially on another physical CPU even if we 1219 * observe only one virtual CPU as the guest. 1220 * 1221 * Prior to MIPS-II, there was no SYNC instruction.[1] CPUs 1222 * with only MIPS-I presumably don't exist in multiprocessor 1223 * configurations. But what if we're running a _kernel_ built 1224 * for a uniprocessor MIPS-I CPU, as a virtual machine guest of 1225 * a _host_ with a newer multiprocessor CPU? How do we enforce 1226 * store-before-load ordering for a paravirtualized device 1227 * driver, coordinating with host software `device' potentially 1228 * on another CPU? You'll have to answer that before you can 1229 * use virtio drivers! 1230 * 1231 * [1] MIPS32 Architecture For Programmers, Volume II: The 1232 * MIPS32 Instruction Set, Document Number: MD00086, 1233 * Revision 0.95, March 12, 2001, MIPS Technologies, p. 215 1234 */ 1235 __asm volatile( 1236 ".set push" "\n\t" 1237 ".set mips2" "\n\t" 1238 "sync" "\n\t" 1239 ".set pop"); 1240 } 1241 #endif /* !MIPS1 */ 1242
Indexes created Sun Oct 25 00:26:45 UTC 2026