1 /* $NetBSD: cpu_subr.c,v 1.6 2025/09/06 02:53:23 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Nick Hudson 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include "opt_ddb.h" 33 #include "opt_multiprocessor.h" 34 #include "opt_riscv_debug.h" 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.6 2025/09/06 02:53:23 riastradh Exp $"); 38 39 #include <sys/param.h> 40 #include <sys/atomic.h> 41 #include <sys/cpu.h> 42 #include <sys/kernel.h> 43 #include <sys/reboot.h> 44 #include <sys/paravirt_membar.h> 45 #include <sys/xcall.h> 46 47 #include <machine/db_machdep.h> 48 #include <machine/sbi.h> 49 50 #ifdef DDB 51 #include <ddb/db_output.h> 52 #endif 53 54 #ifdef VERBOSE_INIT_RISCV 55 #define VPRINTF(...) printf(__VA_ARGS__) 56 #else 57 #define VPRINTF(...) __nothing 58 #endif 59 60 unsigned int cpu_hartindex[MAXCPUS] = { 61 [0 ... MAXCPUS - 1] = ~0U, 62 }; 63 64 cpuid_t cpu_bphartid = ~0UL; 65 66 #ifdef MULTIPROCESSOR 67 68 kcpuset_t *cpus_halted; 69 kcpuset_t *cpus_hatched; 70 kcpuset_t *cpus_paused; 71 kcpuset_t *cpus_resumed; 72 kcpuset_t *cpus_running; 73 74 #define CPUINDEX_DIVISOR (sizeof(u_long) * NBBY) 75 76 #define N howmany(MAXCPUS, CPUINDEX_DIVISOR) 77 78 /* cpu_hatch_ipi needs fixing for > 1 */ 79 CTASSERT(N == 1); 80 volatile u_long riscv_cpu_hatched[N] __cacheline_aligned = { }; 81 volatile u_long riscv_cpu_mbox[N] __cacheline_aligned = { }; 82 83 /* IPI all APs to GO! */ 84 static void 85 cpu_ipi_aps(void) 86 { 87 unsigned long hartmask = 0; 88 89 // BP is index 0 90 for (size_t i = 1; i < ncpu; i++) { 91 const struct cpu_info * const ci = &cpu_info_store[i]; 92 const cpuid_t hartid = ci->ci_cpuid; 93 KASSERT(hartid < sizeof(unsigned long) * NBBY); 94 hartmask |= __BIT(hartid); 95 } 96 struct sbiret sbiret = sbi_send_ipi(hartmask, 0); 97 98 KASSERT(sbiret.error == SBI_SUCCESS); 99 } 100 101 void 102 cpu_boot_secondary_processors(void) 103 { 104 if ((boothowto & RB_MD1) != 0) 105 return; 106 107 VPRINTF("%s: starting secondary processors\n", __func__); 108 109 /* 110 * send mbox to have secondary processors do cpu_hatch() 111 * store-release matches locore.S 112 */ 113 asm volatile("fence rw,w"); 114 for (size_t n = 0; n < __arraycount(riscv_cpu_mbox); n++) 115 atomic_or_ulong(&riscv_cpu_mbox[n], riscv_cpu_hatched[n]); 116 cpu_ipi_aps(); 117 118 /* wait for all cpus to have done cpu_hatch() */ 119 for (u_int cpuindex = 1; cpuindex < ncpu; cpuindex++) { 120 if (!cpu_hatched_p(cpuindex)) 121 continue; 122 123 const size_t off = cpuindex / CPUINDEX_DIVISOR; 124 const u_long bit = __BIT(cpuindex % CPUINDEX_DIVISOR); 125 126 /* load-acquire matches cpu_clr_mbox */ 127 while (atomic_load_acquire(&riscv_cpu_mbox[off]) & bit) { 128 /* spin - it shouldn't be long */ 129 ; 130 } 131 struct cpu_info *ci = &cpu_info_store[cpuindex]; 132 atomic_or_ulong(&ci->ci_flags, CPUF_RUNNING); 133 kcpuset_set(cpus_running, cpu_index(ci)); 134 } 135 136 VPRINTF("%s: secondary processors hatched. %d running\n", __func__, 137 kcpuset_countset(cpus_running)); 138 } 139 140 bool 141 cpu_hatched_p(u_int cpuindex) 142 { 143 const u_int off = cpuindex / CPUINDEX_DIVISOR; 144 const u_int bit = cpuindex % CPUINDEX_DIVISOR; 145 146 /* load-acquire matches cpu_set_hatched */ 147 return (atomic_load_acquire(&riscv_cpu_hatched[off]) & __BIT(bit)) != 0; 148 } 149 150 151 void 152 cpu_set_hatched(u_int cpuindex) 153 { 154 155 const size_t off = cpuindex / CPUINDEX_DIVISOR; 156 const u_long bit = __BIT(cpuindex % CPUINDEX_DIVISOR); 157 158 /* store-release matches cpu_hatched_p */ 159 asm volatile("fence rw, w" ::: "memory"); 160 atomic_or_ulong(&riscv_cpu_hatched[off], bit); 161 162 asm volatile("fence w, rw" ::: "memory"); 163 } 164 165 void 166 cpu_clr_mbox(u_int cpuindex) 167 { 168 169 const size_t off = cpuindex / CPUINDEX_DIVISOR; 170 const u_long bit = __BIT(cpuindex % CPUINDEX_DIVISOR); 171 172 /* store-release matches locore.S */ 173 asm volatile("fence rw,w" ::: "memory"); 174 atomic_and_ulong(&riscv_cpu_mbox[off], ~bit); 175 176 asm volatile("fence w, rw" ::: "memory"); 177 } 178 179 180 void 181 cpu_broadcast_ipi(int tag) 182 { 183 184 /* 185 * No reason to remove ourselves since multicast_ipi will do that 186 * for us. 187 */ 188 cpu_multicast_ipi(cpus_running, tag); 189 } 190 191 void 192 cpu_multicast_ipi(const kcpuset_t *kcp, int tag) 193 { 194 struct cpu_info * const ci = curcpu(); 195 kcpuset_t *kcp2 = ci->ci_multicastcpus; 196 197 if (kcpuset_match(cpus_running, ci->ci_data.cpu_kcpuset)) 198 return; 199 200 kcpuset_copy(kcp2, kcp); 201 kcpuset_remove(kcp2, ci->ci_data.cpu_kcpuset); 202 for (unsigned int cii; (cii = kcpuset_ffs(kcp2)) != 0; ) { 203 kcpuset_clear(kcp2, --cii); 204 (void)cpu_send_ipi(cpu_lookup(cii), tag); 205 } 206 } 207 208 static void 209 cpu_ipi_wait(const char *s, const kcpuset_t *watchset, const kcpuset_t *wanted) 210 { 211 bool done = false; 212 struct cpu_info * const ci = curcpu(); 213 kcpuset_t *kcp = ci->ci_watchcpus; 214 215 /* some finite amount of time */ 216 for (u_long limit = ci->ci_cpu_freq /* / 10 */; !done && limit--; ) { 217 kcpuset_copy(kcp, watchset); 218 kcpuset_intersect(kcp, wanted); 219 done = kcpuset_match(kcp, wanted); 220 } 221 222 if (!done) { 223 cpuid_t cii; 224 kcpuset_copy(kcp, wanted); 225 kcpuset_remove(kcp, watchset); 226 if ((cii = kcpuset_ffs(kcp)) != 0) { 227 printf("Failed to %s:", s); 228 do { 229 kcpuset_clear(kcp, --cii); 230 printf(" cpu%lu", cii); 231 } while ((cii = kcpuset_ffs(kcp)) != 0); 232 printf("\n"); 233 } 234 } 235 } 236 237 /* 238 * Halt this cpu 239 */ 240 void 241 cpu_halt(void) 242 { 243 cpuid_t cii = cpu_index(curcpu()); 244 245 printf("cpu%lu: shutting down\n", cii); 246 kcpuset_atomic_set(cpus_halted, cii); 247 spl0(); /* allow interrupts e.g. further ipi ? */ 248 for (;;) ; /* spin */ 249 250 /* NOTREACHED */ 251 } 252 253 /* 254 * Halt all running cpus, excluding current cpu. 255 */ 256 void 257 cpu_halt_others(void) 258 { 259 kcpuset_t *kcp; 260 261 // If we are the only CPU running, there's nothing to do. 262 if (kcpuset_match(cpus_running, curcpu()->ci_data.cpu_kcpuset)) 263 return; 264 265 // Get all running CPUs 266 kcpuset_clone(&kcp, cpus_running); 267 // Remove ourself 268 kcpuset_remove(kcp, curcpu()->ci_data.cpu_kcpuset); 269 // Remove any halted CPUs 270 kcpuset_remove(kcp, cpus_halted); 271 // If there are CPUs left, send the IPIs 272 if (!kcpuset_iszero(kcp)) { 273 cpu_multicast_ipi(kcp, IPI_HALT); 274 cpu_ipi_wait("halt", cpus_halted, kcp); 275 } 276 kcpuset_destroy(kcp); 277 278 /* 279 * TBD 280 * Depending on available firmware methods, other cpus will 281 * either shut down themselves, or spin and wait for us to 282 * stop them. 283 */ 284 } 285 286 /* 287 * Pause this cpu 288 */ 289 void 290 cpu_pause(void) 291 { 292 const int s = splhigh(); 293 cpuid_t cii = cpu_index(curcpu()); 294 295 if (__predict_false(cold)) { 296 splx(s); 297 return; 298 } 299 300 do { 301 kcpuset_atomic_set(cpus_paused, cii); 302 do { 303 ; 304 } while (kcpuset_isset(cpus_paused, cii)); 305 kcpuset_atomic_set(cpus_resumed, cii); 306 #if defined(DDB) 307 if (ddb_running_on_this_cpu_p()) 308 cpu_Debugger(); 309 if (ddb_running_on_any_cpu_p()) 310 continue; 311 #endif 312 } while (false); 313 314 splx(s); 315 } 316 317 /* 318 * Pause all running cpus, excluding current cpu. 319 */ 320 void 321 cpu_pause_others(void) 322 { 323 struct cpu_info * const ci = curcpu(); 324 325 if (cold || kcpuset_match(cpus_running, ci->ci_data.cpu_kcpuset)) 326 return; 327 328 kcpuset_t *kcp = ci->ci_ddbcpus; 329 330 kcpuset_copy(kcp, cpus_running); 331 kcpuset_remove(kcp, ci->ci_data.cpu_kcpuset); 332 kcpuset_remove(kcp, cpus_paused); 333 334 cpu_broadcast_ipi(IPI_SUSPEND); 335 cpu_ipi_wait("pause", cpus_paused, kcp); 336 } 337 338 /* 339 * Resume a single cpu 340 */ 341 void 342 cpu_resume(cpuid_t cii) 343 { 344 345 if (__predict_false(cold)) 346 return; 347 348 struct cpu_info * const ci = curcpu(); 349 kcpuset_t *kcp = ci->ci_ddbcpus; 350 351 kcpuset_zero(kcp); 352 kcpuset_set(kcp, cii); 353 kcpuset_atomicly_remove(cpus_resumed, cpus_resumed); 354 kcpuset_atomic_clear(cpus_paused, cii); 355 356 cpu_ipi_wait("resume", cpus_resumed, kcp); 357 } 358 359 /* 360 * Resume all paused cpus. 361 */ 362 void 363 cpu_resume_others(void) 364 { 365 366 if (__predict_false(cold)) 367 return; 368 369 struct cpu_info * const ci = curcpu(); 370 kcpuset_t *kcp = ci->ci_ddbcpus; 371 372 kcpuset_atomicly_remove(cpus_resumed, cpus_resumed); 373 kcpuset_copy(kcp, cpus_paused); 374 kcpuset_atomicly_remove(cpus_paused, cpus_paused); 375 376 /* CPUs awake on cpus_paused clear */ 377 cpu_ipi_wait("resume", cpus_resumed, kcp); 378 } 379 380 bool 381 cpu_is_paused(cpuid_t cii) 382 { 383 384 return !cold && kcpuset_isset(cpus_paused, cii); 385 } 386 387 #ifdef DDB 388 void 389 cpu_debug_dump(void) 390 { 391 CPU_INFO_ITERATOR cii; 392 struct cpu_info *ci; 393 char running, hatched, paused, resumed, halted; 394 db_printf("CPU CPUID STATE CPUINFO CPL INT MTX IPIS(A/R)\n"); 395 for (CPU_INFO_FOREACH(cii, ci)) { 396 hatched = (kcpuset_isset(cpus_hatched, cpu_index(ci)) ? 'H' : '-'); 397 running = (kcpuset_isset(cpus_running, cpu_index(ci)) ? 'R' : '-'); 398 paused = (kcpuset_isset(cpus_paused, cpu_index(ci)) ? 'P' : '-'); 399 resumed = (kcpuset_isset(cpus_resumed, cpu_index(ci)) ? 'r' : '-'); 400 halted = (kcpuset_isset(cpus_halted, cpu_index(ci)) ? 'h' : '-'); 401 db_printf("%3d 0x%03lx%c%c%c%c%c%c%c %p " 402 "%3d %3d %3d 0x%02lx/0x%02lx\n", 403 cpu_index(ci), ci->ci_cpuid, 404 ci == curcpu() ? '<' : ' ', 405 CPU_IS_PRIMARY(ci) ? '*' : ' ', 406 hatched, running, paused, resumed, halted, 407 ci, ci->ci_cpl, ci->ci_intr_depth, ci->ci_mtx_count, 408 ci->ci_active_ipis, ci->ci_request_ipis); 409 } 410 } 411 #endif 412 413 void 414 xc_send_ipi(struct cpu_info *ci) 415 { 416 KASSERT(kpreempt_disabled()); 417 KASSERT(curcpu() != ci); 418 419 cpu_send_ipi(ci, IPI_XCALL); 420 } 421 422 void 423 cpu_ipi(struct cpu_info *ci) 424 { 425 KASSERT(kpreempt_disabled()); 426 KASSERT(curcpu() != ci); 427 428 cpu_send_ipi(ci, IPI_GENERIC); 429 } 430 431 #endif 432 433 void 434 paravirt_membar_sync(void) 435 { 436 437 /* 438 * Store-before-load ordering with respect to matching logic 439 * on the hypervisor side. 440 * 441 * This is the same as membar_sync, but guaranteed never to be 442 * conditionalized or hotpatched away even on uniprocessor 443 * builds and boots -- because under virtualization, we still 444 * have to coordinate with a `device' backed by a hypervisor 445 * that is potentially on another physical CPU even if we 446 * observe only one virtual CPU as the guest. 447 */ 448 __asm volatile("fence rw,rw"); 449 } 450
Indexes created Sat Sep 20 22:09:52 GMT 2025