1 /* $NetBSD: subr_xcall.c,v 1.40 2026/07/17 02:16:16 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2007-2010, 2019 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Cross call support 34 * 35 * Background 36 * 37 * Sometimes it is necessary to modify hardware state that is tied 38 * directly to individual CPUs (such as a CPU's local timer), and 39 * these updates can not be done remotely by another CPU. The LWP 40 * requesting the update may be unable to guarantee that it will be 41 * running on the CPU where the update must occur, when the update 42 * occurs. 43 * 44 * Additionally, it's sometimes necessary to modify per-CPU software 45 * state from a remote CPU. Where these update operations are so 46 * rare or the access to the per-CPU data so frequent that the cost 47 * of using locking or atomic operations to provide coherency is 48 * prohibitive, another way must be found. 49 * 50 * Cross calls help to solve these types of problem by allowing 51 * any LWP in the system to request that an arbitrary function be 52 * executed on a specific CPU. 53 * 54 * Implementation 55 * 56 * A slow mechanism for making low priority cross calls is 57 * provided. The function to be executed runs on the remote CPU 58 * within a bound kthread. No queueing is provided, and the 59 * implementation uses global state. The function being called may 60 * block briefly on locks, but in doing so must be careful to not 61 * interfere with other cross calls in the system. The function is 62 * called with thread context and not from a soft interrupt, so it 63 * can ensure that it is not interrupting other code running on the 64 * CPU, and so has exclusive access to the CPU. Since this facility 65 * is heavyweight, it's expected that it will not be used often. 66 * 67 * Cross calls must not allocate memory, as the pagedaemon uses cross 68 * calls (and memory allocation may need to wait on the pagedaemon). 69 * 70 * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is 71 * also provided. The function to be executed runs in software 72 * interrupt context at IPL_SOFTSERIAL level, and is expected to 73 * be very lightweight, e.g. avoid blocking. 74 */ 75 76 #ifdef _KERNEL_OPT 77 #include "opt_multiprocessor.h" 78 #endif 79 80 #include <sys/cdefs.h> 81 __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.40 2026/07/17 02:16:16 thorpej Exp $"); 82 83 #include <sys/types.h> 84 #include <sys/param.h> 85 #include <sys/xcall.h> 86 #include <sys/mutex.h> 87 #include <sys/condvar.h> 88 #include <sys/evcnt.h> 89 #include <sys/kthread.h> 90 #include <sys/cpu.h> 91 #include <sys/atomic.h> 92 93 #ifdef _RUMPKERNEL 94 #include "rump_private.h" 95 #endif 96 97 #if !defined(MULTIPROCESSOR) && !defined(_RUMPKERNEL) 98 #define XCALL_SLIM 99 #endif 100 101 /* Cross-call state box. */ 102 typedef struct { 103 kmutex_t xc_lock; 104 kcondvar_t xc_busy; 105 xcfunc_t xc_func; 106 void * xc_arg1; 107 void * xc_arg2; 108 uint64_t xc_headp; 109 uint64_t xc_donep; 110 unsigned int xc_ipl; 111 } xc_state_t; 112 113 /* Bit indicating high (1) or low (0) priority. */ 114 #define XC_PRI_BIT (1ULL << 63) 115 116 #ifndef XCALL_SLIM 117 /* Low priority xcall structures. */ 118 static xc_state_t xc_low_pri __cacheline_aligned; 119 120 /* High priority xcall structures. */ 121 static xc_state_t xc_high_pri __cacheline_aligned; 122 static void * xc_sihs[4] __cacheline_aligned; 123 124 /* Event counters. */ 125 static struct evcnt xc_unicast_ev __cacheline_aligned; 126 static struct evcnt xc_broadcast_ev __cacheline_aligned; 127 128 static void xc_init(void); 129 static void xc_thread(void *); 130 131 static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *, 132 unsigned int); 133 static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *); 134 #endif /* ! XCALL_SLIM */ 135 136 /* The internal form of IPL */ 137 #define XC_IPL_MASK 0xff00 138 /* 139 * Assign 0 to XC_IPL_SOFTSERIAL to treat IPL_SOFTSERIAL as the default value 140 * (just XC_HIGHPRI). 141 */ 142 #define XC_IPL_SOFTSERIAL 0 143 #define XC_IPL_SOFTNET 1 144 #define XC_IPL_SOFTBIO 2 145 #define XC_IPL_SOFTCLOCK 3 146 #define XC_IPL_MAX XC_IPL_SOFTCLOCK 147 148 #ifndef XCALL_SLIM 149 CTASSERT(XC_IPL_MAX <= __arraycount(xc_sihs)); 150 151 /* 152 * xc_init: 153 * 154 * Initialize low and high priority cross-call structures. 155 */ 156 static void 157 xc_init(void) 158 { 159 xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri; 160 161 memset(xclo, 0, sizeof(xc_state_t)); 162 mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE); 163 cv_init(&xclo->xc_busy, "xclow"); 164 165 memset(xchi, 0, sizeof(xc_state_t)); 166 mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTSERIAL); 167 cv_init(&xchi->xc_busy, "xchigh"); 168 169 /* Set up a softint for each IPL_SOFT*. */ 170 #define SETUP_SOFTINT(xipl, sipl) do { \ 171 xc_sihs[(xipl)] = softint_establish( (sipl) | SOFTINT_MPSAFE,\ 172 xc__highpri_intr, NULL); \ 173 KASSERT(xc_sihs[(xipl)] != NULL); \ 174 } while (0) 175 176 SETUP_SOFTINT(XC_IPL_SOFTSERIAL, SOFTINT_SERIAL); 177 /* 178 * If a IPL_SOFTXXX have the same value of the previous, we don't use 179 * the IPL (see xc_encode_ipl). So we don't need to allocate a softint 180 * for it. 181 */ 182 #if IPL_SOFTNET != IPL_SOFTSERIAL 183 SETUP_SOFTINT(XC_IPL_SOFTNET, SOFTINT_NET); 184 #endif 185 #if IPL_SOFTBIO != IPL_SOFTNET 186 SETUP_SOFTINT(XC_IPL_SOFTBIO, SOFTINT_BIO); 187 #endif 188 #if IPL_SOFTCLOCK != IPL_SOFTBIO 189 SETUP_SOFTINT(XC_IPL_SOFTCLOCK, SOFTINT_CLOCK); 190 #endif 191 192 #undef SETUP_SOFTINT 193 194 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 195 "crosscall", "unicast"); 196 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 197 "crosscall", "broadcast"); 198 } 199 #endif /* ! XCALL_SLIM */ 200 201 /* 202 * Encode an IPL to a form that can be embedded into flags of xc_broadcast 203 * or xc_unicast. 204 */ 205 unsigned int 206 xc_encode_ipl(int ipl) 207 { 208 209 switch (ipl) { 210 case IPL_SOFTSERIAL: 211 return __SHIFTIN(XC_IPL_SOFTSERIAL, XC_IPL_MASK); 212 /* IPL_SOFT* can be the same value (e.g., on sparc or mips). */ 213 #if IPL_SOFTNET != IPL_SOFTSERIAL 214 case IPL_SOFTNET: 215 return __SHIFTIN(XC_IPL_SOFTNET, XC_IPL_MASK); 216 #endif 217 #if IPL_SOFTBIO != IPL_SOFTNET 218 case IPL_SOFTBIO: 219 return __SHIFTIN(XC_IPL_SOFTBIO, XC_IPL_MASK); 220 #endif 221 #if IPL_SOFTCLOCK != IPL_SOFTBIO 222 case IPL_SOFTCLOCK: 223 return __SHIFTIN(XC_IPL_SOFTCLOCK, XC_IPL_MASK); 224 #endif 225 } 226 227 panic("Invalid IPL: %d", ipl); 228 } 229 230 #ifndef XCALL_SLIM 231 /* 232 * Extract an XC_IPL from flags of xc_broadcast or xc_unicast. 233 */ 234 static inline unsigned int 235 xc_extract_ipl(unsigned int flags) 236 { 237 238 return __SHIFTOUT(flags, XC_IPL_MASK); 239 } 240 #endif 241 242 /* 243 * xc_init_cpu: 244 * 245 * Initialize the cross-call subsystem. Called once for each CPU 246 * in the system as they are attached. 247 */ 248 void 249 xc_init_cpu(struct cpu_info *ci) 250 { 251 #ifndef XCALL_SLIM 252 static bool again = false; 253 int error __diagused; 254 255 if (!again) { 256 /* Autoconfiguration will prevent re-entry. */ 257 xc_init(); 258 again = true; 259 } 260 cv_init(&ci->ci_data.cpu_xcall, "xcall"); 261 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 262 NULL, NULL, "xcall/%u", ci->ci_index); 263 KASSERT(error == 0); 264 #endif /* ! XCALL_SLIM */ 265 } 266 267 /* 268 * xc_broadcast: 269 * 270 * Trigger a call on all CPUs in the system. 271 */ 272 uint64_t 273 xc_broadcast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2) 274 { 275 276 KASSERT(!cpu_intr_p()); 277 KASSERT(!cpu_softintr_p()); 278 ASSERT_SLEEPABLE(); 279 280 #ifdef XCALL_SLIM 281 if (flags & XC_HIGHPRI) { 282 int s = splsoftserial(); 283 (*func)(arg1, arg2); 284 splx(s); 285 } else { 286 (*func)(arg1, arg2); 287 } 288 return 0; 289 #else 290 if (__predict_false(!mp_online)) { 291 int s, bound; 292 293 if (flags & XC_HIGHPRI) 294 s = splsoftserial(); 295 else 296 bound = curlwp_bind(); 297 (*func)(arg1, arg2); 298 if (flags & XC_HIGHPRI) 299 splx(s); 300 else 301 curlwp_bindx(bound); 302 return 0; 303 } 304 305 if ((flags & XC_HIGHPRI) != 0) { 306 unsigned int ipl = xc_extract_ipl(flags); 307 return xc_highpri(func, arg1, arg2, NULL, ipl); 308 } else { 309 return xc_lowpri(func, arg1, arg2, NULL); 310 } 311 #endif /* XCALL_SLIM */ 312 } 313 314 #ifndef XCALL_SLIM 315 static void 316 xc_nop(void *arg1, void *arg2) 317 { 318 319 return; 320 } 321 #endif 322 323 /* 324 * xc_barrier: 325 * 326 * Broadcast a nop to all CPUs in the system. 327 */ 328 void 329 xc_barrier(unsigned int flags) 330 { 331 #ifdef XCALL_SLIM 332 (void)flags; 333 #else 334 uint64_t where; 335 336 where = xc_broadcast(flags, xc_nop, NULL, NULL); 337 xc_wait(where); 338 #endif 339 } 340 341 /* 342 * xc_unicast: 343 * 344 * Trigger a call on one CPU. 345 */ 346 uint64_t 347 xc_unicast(unsigned int flags, xcfunc_t func, void *arg1, void *arg2, 348 struct cpu_info *ci) 349 { 350 351 KASSERT(ci != NULL); 352 353 #ifdef XCALL_SLIM 354 KASSERT(ci == curcpu()); 355 return xc_broadcast(flags, func, arg1, arg2); 356 #else 357 KASSERT(!cpu_intr_p()); 358 KASSERT(!cpu_softintr_p()); 359 ASSERT_SLEEPABLE(); 360 361 if (__predict_false(!mp_online)) { 362 int s, bound; 363 364 KASSERT(ci == curcpu()); 365 366 if (flags & XC_HIGHPRI) 367 s = splsoftserial(); 368 else 369 bound = curlwp_bind(); 370 (*func)(arg1, arg2); 371 if (flags & XC_HIGHPRI) 372 splx(s); 373 else 374 curlwp_bindx(bound); 375 376 return 0; 377 } 378 379 if ((flags & XC_HIGHPRI) != 0) { 380 unsigned int ipl = xc_extract_ipl(flags); 381 return xc_highpri(func, arg1, arg2, ci, ipl); 382 } else { 383 return xc_lowpri(func, arg1, arg2, ci); 384 } 385 #endif /* XCALL_SLIM */ 386 } 387 388 /* 389 * xc_wait: 390 * 391 * Wait for a cross call to complete. 392 */ 393 void 394 xc_wait(uint64_t where) 395 { 396 397 KASSERT(!cpu_intr_p()); 398 KASSERT(!cpu_softintr_p()); 399 ASSERT_SLEEPABLE(); 400 401 #ifdef XCALL_SLIM 402 (void)where; 403 #else 404 xc_state_t *xc; 405 406 if (__predict_false(!mp_online)) { 407 return; 408 } 409 410 /* Determine whether it is high or low priority cross-call. */ 411 if ((where & XC_PRI_BIT) != 0) { 412 xc = &xc_high_pri; 413 where &= ~XC_PRI_BIT; 414 } else { 415 xc = &xc_low_pri; 416 } 417 418 #ifdef __HAVE_ATOMIC64_LOADSTORE 419 /* Fast path, if already done. */ 420 if (atomic_load_acquire(&xc->xc_donep) >= where) { 421 return; 422 } 423 #endif 424 425 /* Slow path: block until awoken. */ 426 mutex_enter(&xc->xc_lock); 427 while (xc->xc_donep < where) { 428 cv_wait(&xc->xc_busy, &xc->xc_lock); 429 } 430 mutex_exit(&xc->xc_lock); 431 #endif /* XCALL_SLIM */ 432 } 433 434 #ifndef XCALL_SLIM 435 /* 436 * xc_lowpri: 437 * 438 * Trigger a low priority call on one or more CPUs. 439 */ 440 static inline uint64_t 441 xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) 442 { 443 xc_state_t *xc = &xc_low_pri; 444 CPU_INFO_ITERATOR cii; 445 uint64_t where; 446 447 mutex_enter(&xc->xc_lock); 448 while (xc->xc_headp != xc->xc_donep) { 449 cv_wait(&xc->xc_busy, &xc->xc_lock); 450 } 451 xc->xc_arg1 = arg1; 452 xc->xc_arg2 = arg2; 453 xc->xc_func = func; 454 if (ci == NULL) { 455 xc_broadcast_ev.ev_count++; 456 for (CPU_INFO_FOREACH(cii, ci)) { 457 if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0) 458 continue; 459 xc->xc_headp += 1; 460 ci->ci_data.cpu_xcall_pending = true; 461 cv_signal(&ci->ci_data.cpu_xcall); 462 } 463 } else { 464 xc_unicast_ev.ev_count++; 465 xc->xc_headp += 1; 466 ci->ci_data.cpu_xcall_pending = true; 467 cv_signal(&ci->ci_data.cpu_xcall); 468 } 469 KASSERT(xc->xc_donep < xc->xc_headp); 470 where = xc->xc_headp; 471 mutex_exit(&xc->xc_lock); 472 473 /* Return a low priority ticket. */ 474 KASSERT((where & XC_PRI_BIT) == 0); 475 return where; 476 } 477 478 /* 479 * xc_thread: 480 * 481 * One thread per-CPU to dispatch low priority calls. 482 */ 483 static void 484 xc_thread(void *cookie) 485 { 486 struct cpu_info *ci = curcpu(); 487 xc_state_t *xc = &xc_low_pri; 488 void *arg1, *arg2; 489 xcfunc_t func; 490 struct lwp *l = curlwp; 491 492 KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d", 493 l, l->l_nopreempt); 494 495 mutex_enter(&xc->xc_lock); 496 for (;;) { 497 while (!ci->ci_data.cpu_xcall_pending) { 498 if (xc->xc_headp == xc->xc_donep) { 499 cv_broadcast(&xc->xc_busy); 500 } 501 cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock); 502 KASSERT(ci == curcpu()); 503 } 504 ci->ci_data.cpu_xcall_pending = false; 505 func = xc->xc_func; 506 arg1 = xc->xc_arg1; 507 arg2 = xc->xc_arg2; 508 mutex_exit(&xc->xc_lock); 509 510 KASSERT(func != NULL); 511 (*func)(arg1, arg2); 512 513 KASSERTMSG(l->l_nopreempt == 0, "lwp %p nopreempt %d func %p", 514 l, l->l_nopreempt, func); 515 516 mutex_enter(&xc->xc_lock); 517 #ifdef __HAVE_ATOMIC64_LOADSTORE 518 atomic_store_release(&xc->xc_donep, xc->xc_donep + 1); 519 #else 520 xc->xc_donep++; 521 #endif 522 } 523 /* NOTREACHED */ 524 } 525 #endif /* ! XCALL_SLIM */ 526 527 /* 528 * xc_ipi_handler: 529 * 530 * Handler of cross-call IPI. 531 */ 532 void 533 xc_ipi_handler(void) 534 { 535 #ifndef XCALL_SLIM 536 xc_state_t *xc = & xc_high_pri; 537 538 KASSERT(xc->xc_ipl < __arraycount(xc_sihs)); 539 KASSERT(xc_sihs[xc->xc_ipl] != NULL); 540 541 /* Executes xc__highpri_intr() via software interrupt. */ 542 softint_schedule(xc_sihs[xc->xc_ipl]); 543 #endif 544 } 545 546 /* 547 * xc__highpri_intr: 548 * 549 * A software interrupt handler for high priority calls. 550 */ 551 void 552 xc__highpri_intr(void *dummy) 553 { 554 #ifdef XCALL_SLIM 555 (void)dummy; 556 #else 557 xc_state_t *xc = &xc_high_pri; 558 void *arg1, *arg2; 559 xcfunc_t func; 560 561 KASSERTMSG(!cpu_intr_p(), "high priority xcall for function %p", 562 xc->xc_func); 563 /* 564 * Lock-less fetch of function and its arguments. 565 * Safe since it cannot change at this point. 566 */ 567 func = xc->xc_func; 568 arg1 = xc->xc_arg1; 569 arg2 = xc->xc_arg2; 570 571 KASSERT(func != NULL); 572 (*func)(arg1, arg2); 573 574 /* 575 * Note the request as done, and if we have reached the head, 576 * cross-call has been processed - notify waiters, if any. 577 */ 578 mutex_enter(&xc->xc_lock); 579 KASSERT(xc->xc_donep < xc->xc_headp); 580 #ifdef __HAVE_ATOMIC64_LOADSTORE 581 atomic_store_release(&xc->xc_donep, xc->xc_donep + 1); 582 #else 583 xc->xc_donep++; 584 #endif 585 if (xc->xc_donep == xc->xc_headp) { 586 cv_broadcast(&xc->xc_busy); 587 } 588 mutex_exit(&xc->xc_lock); 589 #endif /* XCALL_SLIM */ 590 } 591 592 #ifndef XCALL_SLIM 593 /* 594 * xc_highpri: 595 * 596 * Trigger a high priority call on one or more CPUs. 597 */ 598 static inline uint64_t 599 xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci, 600 unsigned int ipl) 601 { 602 xc_state_t *xc = &xc_high_pri; 603 uint64_t where; 604 605 mutex_enter(&xc->xc_lock); 606 while (xc->xc_headp != xc->xc_donep) { 607 cv_wait(&xc->xc_busy, &xc->xc_lock); 608 } 609 xc->xc_func = func; 610 xc->xc_arg1 = arg1; 611 xc->xc_arg2 = arg2; 612 xc->xc_headp += (ci ? 1 : ncpu); 613 xc->xc_ipl = ipl; 614 where = xc->xc_headp; 615 mutex_exit(&xc->xc_lock); 616 617 /* 618 * Send the IPI once lock is released. 619 * Note: it will handle the local CPU case. 620 */ 621 622 #ifdef _RUMPKERNEL 623 rump_xc_highpri(ci); 624 #else 625 #ifdef MULTIPROCESSOR 626 kpreempt_disable(); 627 if (curcpu() == ci) { 628 /* Unicast: local CPU. */ 629 xc_ipi_handler(); 630 } else if (ci) { 631 /* Unicast: remote CPU. */ 632 xc_send_ipi(ci); 633 } else { 634 /* Broadcast: all, including local. */ 635 xc_send_ipi(NULL); 636 xc_ipi_handler(); 637 } 638 kpreempt_enable(); 639 #else 640 KASSERT(ci == NULL || curcpu() == ci); 641 xc_ipi_handler(); 642 #endif 643 #endif 644 645 /* Indicate a high priority ticket. */ 646 return (where | XC_PRI_BIT); 647 } 648 #endif /* ! XCALL_SLIM */ 649
Indexes created Sat Aug 22 00:26:31 UTC 2026