1 /* $NetBSD: octeon_cpunode.c,v 1.22 2022/03/03 06:27:41 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #define __INTR_PRIVATE 32 #include <sys/cdefs.h> 33 34 __KERNEL_RCSID(0, "$NetBSD: octeon_cpunode.c,v 1.22 2022/03/03 06:27:41 riastradh Exp $"); 35 36 #include "locators.h" 37 #include "cpunode.h" 38 #include "opt_multiprocessor.h" 39 #include "opt_ddb.h" 40 41 #include <sys/param.h> 42 #include <sys/atomic.h> 43 #include <sys/cpu.h> 44 #include <sys/device.h> 45 #include <sys/lwp.h> 46 #include <sys/reboot.h> 47 #include <sys/wdog.h> 48 49 #include <uvm/uvm.h> 50 51 #include <dev/sysmon/sysmonvar.h> 52 53 #include <mips/cache.h> 54 #include <mips/mips_opcode.h> 55 #include <mips/mips3_clock.h> 56 #include <mips/mips3_pte.h> 57 58 #include <mips/cavium/octeonvar.h> 59 #include <mips/cavium/dev/octeon_ciureg.h> 60 #include <mips/cavium/dev/octeon_corereg.h> 61 62 extern struct cpu_softc octeon_cpu_softc[]; 63 64 struct cpunode_attach_args { 65 const char *cnaa_name; 66 int cnaa_cpunum; 67 }; 68 69 struct cpunode_softc { 70 device_t sc_dev; 71 device_t sc_wdog_dev; 72 }; 73 74 static int cpunode_mainbus_match(device_t, cfdata_t, void *); 75 static void cpunode_mainbus_attach(device_t, device_t, void *); 76 77 static int cpu_cpunode_match(device_t, cfdata_t, void *); 78 static void cpu_cpunode_attach(device_t, device_t, void *); 79 80 CFATTACH_DECL_NEW(cpunode, sizeof(struct cpunode_softc), 81 cpunode_mainbus_match, cpunode_mainbus_attach, NULL, NULL); 82 83 CFATTACH_DECL_NEW(cpu_cpunode, 0, 84 cpu_cpunode_match, cpu_cpunode_attach, NULL, NULL); 85 86 #ifdef MULTIPROCESSOR 87 CTASSERT(MAXCPUS <= sizeof(uint64_t) * NBBY); 88 volatile uint64_t cpus_booted = __BIT(0); /* cpu0 is always booted */ 89 #endif 90 91 static void wdog_cpunode_poke(void *arg); 92 93 static int 94 cpunode_mainbus_print(void *aux, const char *pnp) 95 { 96 struct cpunode_attach_args * const cnaa = aux; 97 98 if (pnp) 99 aprint_normal("%s", pnp); 100 101 if (cnaa->cnaa_cpunum != CPUNODECF_CORE_DEFAULT) 102 aprint_normal(" core %d", cnaa->cnaa_cpunum); 103 104 return UNCONF; 105 } 106 107 int 108 cpunode_mainbus_match(device_t parent, cfdata_t cf, void *aux) 109 { 110 111 return 1; 112 } 113 114 void 115 cpunode_mainbus_attach(device_t parent, device_t self, void *aux) 116 { 117 struct cpunode_softc * const sc = device_private(self); 118 const uint64_t fuse = octeon_xkphys_read_8(CIU_FUSE); 119 int cpunum = 0; 120 121 sc->sc_dev = self; 122 123 aprint_naive(": %u core%s\n", popcount64(fuse), fuse == 1 ? "" : "s"); 124 aprint_normal(": %u core%s", popcount64(fuse), fuse == 1 ? "" : "s"); 125 126 const uint64_t cvmctl = mips_cp0_cvmctl_read(); 127 aprint_normal(", %scrypto", (cvmctl & CP0_CVMCTL_NOCRYPTO) ? "no " : ""); 128 aprint_normal((cvmctl & CP0_CVMCTL_KASUMI) ? "+kasumi" : ""); 129 aprint_normal(", %s64bit-mul", (cvmctl & CP0_CVMCTL_NOMUL) ? "no " : ""); 130 if (cvmctl & CP0_CVMCTL_REPUN) 131 aprint_normal(", unaligned-access ok"); 132 #ifdef MULTIPROCESSOR 133 aprint_normal(", booted %#" PRIx64, cpus_booted); 134 #endif 135 aprint_normal("\n"); 136 137 for (uint64_t f = fuse; f != 0; f >>= 1, cpunum++) { 138 struct cpunode_attach_args cnaa = { 139 .cnaa_name = "cpu", 140 .cnaa_cpunum = cpunum, 141 }; 142 config_found(self, &cnaa, cpunode_mainbus_print, CFARGS_NONE); 143 } 144 #if NWDOG > 0 145 struct cpunode_attach_args cnaa = { 146 .cnaa_name = "wdog", 147 .cnaa_cpunum = CPUNODECF_CORE_DEFAULT, 148 }; 149 config_found(self, &cnaa, cpunode_mainbus_print, CFARGS_NONE); 150 #endif 151 } 152 153 int 154 cpu_cpunode_match(device_t parent, cfdata_t cf, void *aux) 155 { 156 struct cpunode_attach_args * const cnaa = aux; 157 const int cpunum = cf->cf_loc[CPUNODECF_CORE]; 158 159 return strcmp(cnaa->cnaa_name, cf->cf_name) == 0 160 && (cpunum == CPUNODECF_CORE_DEFAULT || cpunum == cnaa->cnaa_cpunum); 161 } 162 163 #if defined(MULTIPROCESSOR) 164 static bool 165 octeon_fixup_cpu_info_references(int32_t load_addr, uint32_t new_insns[2], 166 void *arg) 167 { 168 struct cpu_info * const ci = arg; 169 170 atomic_or_ulong(&curcpu()->ci_flags, CPUF_PRESENT); 171 172 KASSERT(MIPS_KSEG0_P(load_addr)); 173 #ifdef MULTIPROCESSOR 174 KASSERT(!CPU_IS_PRIMARY(curcpu())); 175 #endif 176 load_addr += (intptr_t)ci - (intptr_t)&cpu_info_store; 177 178 KASSERT((intptr_t)ci <= load_addr); 179 KASSERT(load_addr < (intptr_t)(ci + 1)); 180 181 KASSERT(INSN_LUI_P(new_insns[0])); 182 KASSERT(INSN_LOAD_P(new_insns[1]) || INSN_STORE_P(new_insns[1])); 183 184 /* 185 * Use the lui and load/store instruction as a prototype and 186 * make it refer to cpu1_info_store instead of cpu_info_store. 187 */ 188 new_insns[0] &= __BITS(31,16); 189 new_insns[1] &= __BITS(31,16); 190 new_insns[0] |= (uint16_t)((load_addr + 0x8000) >> 16); 191 new_insns[1] |= (uint16_t)load_addr; 192 #ifdef DEBUG_VERBOSE 193 printf("%s: %08x: insn#1 %08x: lui r%u, %d\n", 194 __func__, load_addr, new_insns[0], 195 (new_insns[0] >> 16) & 31, 196 (int16_t)new_insns[0]); 197 printf("%s: %08x: insn#2 %08x: %c%c r%u, %d(r%u)\n", 198 __func__, load_addr, new_insns[1], 199 INSN_LOAD_P(new_insns[1]) ? 'l' : 's', 200 INSN_LW_P(new_insns[1]) ? 'w' : 'd', 201 (new_insns[1] >> 16) & 31, 202 (int16_t)new_insns[1], 203 (new_insns[1] >> 21) & 31); 204 #endif 205 return true; 206 } 207 208 static void 209 octeon_cpu_init(struct cpu_info *ci) 210 { 211 extern const mips_locore_jumpvec_t mips64r2_locore_vec; 212 bool ok __diagused; 213 214 mips3_cp0_pg_mask_write(MIPS3_PG_SIZE_TO_MASK(PAGE_SIZE)); 215 mips3_cp0_wired_write(0); 216 (*mips64r2_locore_vec.ljv_tlb_invalidate_all)(); 217 mips3_cp0_wired_write(pmap_tlb0_info.ti_wired); 218 219 // First thing is setup the exception vectors for this cpu. 220 mips64r2_vector_init(&mips_splsw); 221 222 // Next rewrite those exceptions to use this cpu's cpu_info. 223 ok = mips_fixup_exceptions(octeon_fixup_cpu_info_references, ci); 224 KASSERT(ok); 225 226 (void) splhigh(); // make sure interrupts are masked 227 228 KASSERT((mipsNN_cp0_ebase_read() & MIPS_EBASE_CPUNUM) == ci->ci_cpuid); 229 KASSERT(curcpu() == ci); 230 KASSERT(ci->ci_cpl == IPL_HIGH); 231 KASSERT((mips_cp0_status_read() & MIPS_INT_MASK) == 0); 232 } 233 234 static void 235 octeon_cpu_run(struct cpu_info *ci) 236 { 237 238 octeon_intr_init(ci); 239 240 mips3_initclocks(); 241 KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl); 242 KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE); 243 244 aprint_normal("%s: ", device_xname(ci->ci_dev)); 245 cpu_identify(ci->ci_dev); 246 } 247 #endif /* MULTIPROCESSOR */ 248 249 static void 250 cpu_cpunode_attach_common(device_t self, struct cpu_info *ci) 251 { 252 struct cpu_softc * const cpu __diagused = ci->ci_softc; 253 254 ci->ci_dev = self; 255 device_set_private(self, ci); 256 257 KASSERTMSG(cpu != NULL, "ci %p index %d", ci, cpu_index(ci)); 258 259 #if NWDOG > 0 || defined(DDB) 260 /* XXXXXX __mips_n32 and MIPS_PHYS_TO_XKPHYS_CACHED needed here?????? */ 261 void **nmi_vector = (void *)MIPS_PHYS_TO_KSEG0(0x800 + 32*ci->ci_cpuid); 262 *nmi_vector = octeon_reset_vector; 263 264 struct vm_page * const pg = PMAP_ALLOC_POOLPAGE(UVM_PGA_ZERO); 265 KASSERT(pg != NULL); 266 const vaddr_t kva = PMAP_MAP_POOLPAGE(VM_PAGE_TO_PHYS(pg)); 267 KASSERT(kva != 0); 268 ci->ci_nmi_stack = (void *)(kva + PAGE_SIZE - sizeof(struct kernframe)); 269 #endif 270 271 #if NWDOG > 0 272 cpu->cpu_wdog_sih = softint_establish(SOFTINT_CLOCK|SOFTINT_MPSAFE, 273 wdog_cpunode_poke, cpu); 274 KASSERT(cpu->cpu_wdog_sih != NULL); 275 #endif 276 277 aprint_normal(": %lu.%02luMHz\n", 278 (ci->ci_cpu_freq + 5000) / 1000000, 279 ((ci->ci_cpu_freq + 5000) % 1000000) / 10000); 280 aprint_debug_dev(self, "hz cycles = %lu, delay divisor = %lu\n", 281 ci->ci_cycles_per_hz, ci->ci_divisor_delay); 282 283 if (CPU_IS_PRIMARY(ci)) { 284 aprint_normal("%s: ", device_xname(self)); 285 cpu_identify(self); 286 } 287 cpu_attach_common(self, ci); 288 #ifdef MULTIPROCESSOR 289 KASSERT(cpuid_infos[ci->ci_cpuid] == ci); 290 #endif 291 } 292 293 void 294 cpu_cpunode_attach(device_t parent, device_t self, void *aux) 295 { 296 struct cpunode_attach_args * const cnaa = aux; 297 const int cpunum = cnaa->cnaa_cpunum; 298 299 if (cpunum == 0) { 300 cpu_cpunode_attach_common(self, curcpu()); 301 #ifdef MULTIPROCESSOR 302 mips_locoresw.lsw_cpu_init = octeon_cpu_init; 303 mips_locoresw.lsw_cpu_run = octeon_cpu_run; 304 #endif 305 return; 306 } 307 #ifdef MULTIPROCESSOR 308 if ((boothowto & RB_MD1) != 0) { 309 aprint_naive("\n"); 310 aprint_normal(": multiprocessor boot disabled\n"); 311 return; 312 } 313 314 if (!(cpus_booted & __BIT(cpunum))) { 315 aprint_naive(" disabled\n"); 316 aprint_normal(" disabled (unresponsive)\n"); 317 return; 318 } 319 struct cpu_info * const ci = cpu_info_alloc(NULL, cpunum, 0, cpunum, 0); 320 321 ci->ci_softc = &octeon_cpu_softc[cpunum]; 322 ci->ci_softc->cpu_ci = ci; 323 324 cpu_cpunode_attach_common(self, ci); 325 326 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 327 for (int i = 0; i < 100 && !kcpuset_isset(cpus_hatched, cpunum); i++) { 328 delay(10000); 329 } 330 if (!kcpuset_isset(cpus_hatched, cpunum)) { 331 #ifdef DDB 332 aprint_verbose_dev(self, "hatch failed ci=%p flags=%#lx\n", ci, ci->ci_flags); 333 cpu_Debugger(); 334 #endif 335 panic("%s failed to hatch: ci=%p flags=%#lx", 336 cpu_name(ci), ci, ci->ci_flags); 337 } 338 #else 339 aprint_naive(": disabled\n"); 340 aprint_normal(": disabled (uniprocessor kernel)\n"); 341 #endif 342 } 343 344 #if NWDOG > 0 345 struct wdog_softc { 346 struct sysmon_wdog sc_smw; 347 device_t sc_dev; 348 u_int sc_wdog_period; 349 bool sc_wdog_armed; 350 }; 351 352 #ifndef OCTEON_WDOG_PERIOD_DEFAULT 353 #define OCTEON_WDOG_PERIOD_DEFAULT 4 354 #endif 355 356 static int wdog_cpunode_match(device_t, cfdata_t, void *); 357 static void wdog_cpunode_attach(device_t, device_t, void *); 358 359 CFATTACH_DECL_NEW(wdog_cpunode, sizeof(struct wdog_softc), 360 wdog_cpunode_match, wdog_cpunode_attach, NULL, NULL); 361 362 static int 363 wdog_cpunode_setmode(struct sysmon_wdog *smw) 364 { 365 struct wdog_softc * const sc = smw->smw_cookie; 366 367 if ((smw->smw_mode & WDOG_MODE_MASK) == WDOG_MODE_DISARMED) { 368 if (sc->sc_wdog_armed) { 369 CPU_INFO_ITERATOR cii; 370 struct cpu_info *ci; 371 for (CPU_INFO_FOREACH(cii, ci)) { 372 struct cpu_softc * const cpu = ci->ci_softc; 373 uint64_t wdog = mips3_ld(cpu->cpu_wdog); 374 wdog &= ~CIU_WDOGX_MODE; 375 mips3_sd(cpu->cpu_pp_poke, wdog); 376 aprint_verbose_dev(sc->sc_dev, 377 "%s: disable wdog=%#"PRIx64"\n", 378 cpu_name(ci), wdog); 379 mips3_sd(cpu->cpu_wdog, wdog); 380 mips3_sd(cpu->cpu_pp_poke, wdog); 381 } 382 sc->sc_wdog_armed = false; 383 } 384 } else if (!sc->sc_wdog_armed) { 385 kpreempt_disable(); 386 struct cpu_info *ci = curcpu(); 387 if (smw->smw_period == WDOG_PERIOD_DEFAULT) { 388 smw->smw_period = OCTEON_WDOG_PERIOD_DEFAULT; 389 } 390 uint64_t wdog_len = smw->smw_period * ci->ci_cpu_freq; 391 // 392 // This wdog is a 24-bit counter that decrements every 256 393 // cycles. This is then a 32-bit counter so as long wdog_len 394 // doesn't overflow a 32-bit value, we are fine. We write the 395 // 16-bits of the 32-bit period. 396 if ((wdog_len >> 32) != 0) { 397 kpreempt_enable(); 398 return EINVAL; 399 } 400 sc->sc_wdog_period = smw->smw_period; 401 CPU_INFO_ITERATOR cii; 402 for (CPU_INFO_FOREACH(cii, ci)) { 403 struct cpu_softc * const cpu = ci->ci_softc; 404 uint64_t wdog = mips3_ld(cpu->cpu_wdog); 405 wdog &= ~(CIU_WDOGX_MODE|CIU_WDOGX_LEN); 406 wdog |= __SHIFTIN(3, CIU_WDOGX_MODE); 407 wdog |= __SHIFTIN(wdog_len >> 16, CIU_WDOGX_LEN); 408 aprint_verbose_dev(sc->sc_dev, 409 "%s: enable wdog=%#"PRIx64" (%#"PRIx64")\n", 410 cpu_name(ci), wdog, wdog_len); 411 mips3_sd(cpu->cpu_wdog, wdog); 412 } 413 sc->sc_wdog_armed = true; 414 kpreempt_enable(); 415 } 416 return 0; 417 } 418 419 static void 420 wdog_cpunode_poke(void *arg) 421 { 422 struct cpu_softc *cpu = arg; 423 424 mips3_sd(cpu->cpu_pp_poke, 0); 425 } 426 427 static int 428 wdog_cpunode_tickle(struct sysmon_wdog *smw) 429 { 430 431 wdog_cpunode_poke(curcpu()->ci_softc); 432 #ifdef MULTIPROCESSOR 433 // We need to send IPIs to the other CPUs to poke their wdog. 434 cpu_send_ipi(NULL, IPI_WDOG); 435 #endif 436 return 0; 437 } 438 439 int 440 wdog_cpunode_match(device_t parent, cfdata_t cf, void *aux) 441 { 442 struct cpunode_softc * const sc = device_private(parent); 443 struct cpunode_attach_args * const cnaa = aux; 444 const int cpunum = cf->cf_loc[CPUNODECF_CORE]; 445 446 return sc->sc_wdog_dev == NULL 447 && strcmp(cnaa->cnaa_name, cf->cf_name) == 0 448 && cpunum == CPUNODECF_CORE_DEFAULT; 449 } 450 451 void 452 wdog_cpunode_attach(device_t parent, device_t self, void *aux) 453 { 454 struct cpunode_softc * const psc = device_private(parent); 455 struct wdog_softc * const sc = device_private(self); 456 cfdata_t const cf = device_cfdata(self); 457 458 psc->sc_wdog_dev = self; 459 460 sc->sc_dev = self; 461 sc->sc_smw.smw_name = device_xname(self); 462 sc->sc_smw.smw_cookie = sc; 463 sc->sc_smw.smw_setmode = wdog_cpunode_setmode; 464 sc->sc_smw.smw_tickle = wdog_cpunode_tickle; 465 sc->sc_smw.smw_period = OCTEON_WDOG_PERIOD_DEFAULT; 466 sc->sc_wdog_period = sc->sc_smw.smw_period; 467 468 /* 469 * We need one softint per cpu. It's to tickle the softints on 470 * other CPUs. 471 */ 472 #if 0 /* XXX unused? */ 473 CPU_INFO_ITERATOR cii; 474 struct cpu_info *ci; 475 for (CPU_INFO_FOREACH(cii, ci)) { 476 } 477 #endif 478 479 aprint_normal(": default period is %u second%s\n", 480 sc->sc_wdog_period, sc->sc_wdog_period == 1 ? "" : "s"); 481 482 if (sysmon_wdog_register(&sc->sc_smw) != 0) { 483 aprint_error_dev(self, "unable to register with sysmon\n"); 484 return; 485 } 486 487 if (cf->cf_flags & 1) { 488 int error = sysmon_wdog_setmode(&sc->sc_smw, WDOG_MODE_KTICKLE, 489 sc->sc_wdog_period); 490 if (error) 491 aprint_error_dev(self, 492 "failed to start kernel tickler: %d\n", error); 493 } 494 } 495 #endif /* NWDOG > 0 */ 496
Indexes created Fri Sep 26 20:09:58 GMT 2025