subr_xcall.c revision 1.13.16.2
1 /* $NetBSD: subr_xcall.c,v 1.13.16.2 2017/07/06 15:18:23 snj Exp $ */ 2 3 /*- 4 * Copyright (c) 2007-2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran and Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Cross call support 34 * 35 * Background 36 * 37 * Sometimes it is necessary to modify hardware state that is tied 38 * directly to individual CPUs (such as a CPU's local timer), and 39 * these updates can not be done remotely by another CPU. The LWP 40 * requesting the update may be unable to guarantee that it will be 41 * running on the CPU where the update must occur, when the update 42 * occurs. 43 * 44 * Additionally, it's sometimes necessary to modify per-CPU software 45 * state from a remote CPU. Where these update operations are so 46 * rare or the access to the per-CPU data so frequent that the cost 47 * of using locking or atomic operations to provide coherency is 48 * prohibitive, another way must be found. 49 * 50 * Cross calls help to solve these types of problem by allowing 51 * any CPU in the system to request that an arbitrary function be 52 * executed on any other CPU. 53 * 54 * Implementation 55 * 56 * A slow mechanism for making 'low priority' cross calls is 57 * provided. The function to be executed runs on the remote CPU 58 * within a bound kthread. No queueing is provided, and the 59 * implementation uses global state. The function being called may 60 * block briefly on locks, but in doing so must be careful to not 61 * interfere with other cross calls in the system. The function is 62 * called with thread context and not from a soft interrupt, so it 63 * can ensure that it is not interrupting other code running on the 64 * CPU, and so has exclusive access to the CPU. Since this facility 65 * is heavyweight, it's expected that it will not be used often. 66 * 67 * Cross calls must not allocate memory, as the pagedaemon uses 68 * them (and memory allocation may need to wait on the pagedaemon). 69 * 70 * A low-overhead mechanism for high priority calls (XC_HIGHPRI) is 71 * also provided. The function to be executed runs on a software 72 * interrupt context, at SOFTINT_CLOCK level, and is expected to be 73 * very lightweight, e.g. avoid blocking. 74 */ 75 76 #include <sys/cdefs.h> 77 __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.13.16.2 2017/07/06 15:18:23 snj Exp $"); 78 79 #include <sys/types.h> 80 #include <sys/param.h> 81 #include <sys/xcall.h> 82 #include <sys/mutex.h> 83 #include <sys/condvar.h> 84 #include <sys/evcnt.h> 85 #include <sys/kthread.h> 86 #include <sys/cpu.h> 87 88 /* Cross-call state box. */ 89 typedef struct { 90 kmutex_t xc_lock; 91 kcondvar_t xc_busy; 92 xcfunc_t xc_func; 93 void * xc_arg1; 94 void * xc_arg2; 95 uint64_t xc_headp; 96 uint64_t xc_donep; 97 } xc_state_t; 98 99 /* Bit indicating high (1) or low (0) priority. */ 100 #define XC_PRI_BIT (1ULL << 63) 101 102 /* Low priority xcall structures. */ 103 static xc_state_t xc_low_pri __cacheline_aligned; 104 105 /* High priority xcall structures. */ 106 static xc_state_t xc_high_pri __cacheline_aligned; 107 static void * xc_sih __cacheline_aligned; 108 109 /* Event counters. */ 110 static struct evcnt xc_unicast_ev __cacheline_aligned; 111 static struct evcnt xc_broadcast_ev __cacheline_aligned; 112 113 static void xc_init(void); 114 static void xc_thread(void *); 115 static void xc_highpri_intr(void *); 116 117 static inline uint64_t xc_highpri(xcfunc_t, void *, void *, struct cpu_info *); 118 static inline uint64_t xc_lowpri(xcfunc_t, void *, void *, struct cpu_info *); 119 120 /* 121 * xc_init: 122 * 123 * Initialize low and high priority cross-call structures. 124 */ 125 static void 126 xc_init(void) 127 { 128 xc_state_t *xclo = &xc_low_pri, *xchi = &xc_high_pri; 129 130 memset(xclo, 0, sizeof(xc_state_t)); 131 mutex_init(&xclo->xc_lock, MUTEX_DEFAULT, IPL_NONE); 132 cv_init(&xclo->xc_busy, "xclocv"); 133 134 memset(xchi, 0, sizeof(xc_state_t)); 135 mutex_init(&xchi->xc_lock, MUTEX_DEFAULT, IPL_SOFTCLOCK); 136 cv_init(&xchi->xc_busy, "xchicv"); 137 xc_sih = softint_establish(SOFTINT_CLOCK | SOFTINT_MPSAFE, 138 xc_highpri_intr, NULL); 139 KASSERT(xc_sih != NULL); 140 141 evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 142 "crosscall", "unicast"); 143 evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 144 "crosscall", "broadcast"); 145 } 146 147 /* 148 * xc_init_cpu: 149 * 150 * Initialize the cross-call subsystem. Called once for each CPU 151 * in the system as they are attached. 152 */ 153 void 154 xc_init_cpu(struct cpu_info *ci) 155 { 156 static bool again = false; 157 int error; 158 159 if (!again) { 160 /* Autoconfiguration will prevent re-entry. */ 161 xc_init(); 162 again = true; 163 } 164 cv_init(&ci->ci_data.cpu_xcall, "xcall"); 165 error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 166 NULL, NULL, "xcall/%u", ci->ci_index); 167 KASSERT(error == 0); 168 } 169 170 /* 171 * xc_broadcast: 172 * 173 * Trigger a call on all CPUs in the system. 174 */ 175 uint64_t 176 xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) 177 { 178 179 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 180 181 if ((flags & XC_HIGHPRI) != 0) { 182 return xc_highpri(func, arg1, arg2, NULL); 183 } else { 184 return xc_lowpri(func, arg1, arg2, NULL); 185 } 186 } 187 188 /* 189 * xc_unicast: 190 * 191 * Trigger a call on one CPU. 192 */ 193 uint64_t 194 xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, 195 struct cpu_info *ci) 196 { 197 198 KASSERT(ci != NULL); 199 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 200 201 if ((flags & XC_HIGHPRI) != 0) { 202 return xc_highpri(func, arg1, arg2, ci); 203 } else { 204 return xc_lowpri(func, arg1, arg2, ci); 205 } 206 } 207 208 /* 209 * xc_wait: 210 * 211 * Wait for a cross call to complete. 212 */ 213 void 214 xc_wait(uint64_t where) 215 { 216 xc_state_t *xc; 217 218 KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 219 220 /* Determine whether it is high or low priority cross-call. */ 221 if ((where & XC_PRI_BIT) != 0) { 222 xc = &xc_high_pri; 223 where &= ~XC_PRI_BIT; 224 } else { 225 xc = &xc_low_pri; 226 } 227 228 /* Fast path, if already done. */ 229 if (xc->xc_donep >= where) { 230 return; 231 } 232 233 /* Slow path: block until awoken. */ 234 mutex_enter(&xc->xc_lock); 235 while (xc->xc_donep < where) { 236 cv_wait(&xc->xc_busy, &xc->xc_lock); 237 } 238 mutex_exit(&xc->xc_lock); 239 } 240 241 /* 242 * xc_lowpri: 243 * 244 * Trigger a low priority call on one or more CPUs. 245 */ 246 static inline uint64_t 247 xc_lowpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) 248 { 249 xc_state_t *xc = &xc_low_pri; 250 CPU_INFO_ITERATOR cii; 251 uint64_t where; 252 253 mutex_enter(&xc->xc_lock); 254 while (xc->xc_headp != xc->xc_donep) { 255 cv_wait(&xc->xc_busy, &xc->xc_lock); 256 } 257 xc->xc_arg1 = arg1; 258 xc->xc_arg2 = arg2; 259 xc->xc_func = func; 260 if (ci == NULL) { 261 xc_broadcast_ev.ev_count++; 262 for (CPU_INFO_FOREACH(cii, ci)) { 263 if ((ci->ci_schedstate.spc_flags & SPCF_RUNNING) == 0) 264 continue; 265 xc->xc_headp += 1; 266 ci->ci_data.cpu_xcall_pending = true; 267 cv_signal(&ci->ci_data.cpu_xcall); 268 } 269 } else { 270 xc_unicast_ev.ev_count++; 271 xc->xc_headp += 1; 272 ci->ci_data.cpu_xcall_pending = true; 273 cv_signal(&ci->ci_data.cpu_xcall); 274 } 275 KASSERT(xc->xc_donep < xc->xc_headp); 276 where = xc->xc_headp; 277 mutex_exit(&xc->xc_lock); 278 279 /* Return a low priority ticket. */ 280 KASSERT((where & XC_PRI_BIT) == 0); 281 return where; 282 } 283 284 /* 285 * xc_thread: 286 * 287 * One thread per-CPU to dispatch low priority calls. 288 */ 289 static void 290 xc_thread(void *cookie) 291 { 292 struct cpu_info *ci = curcpu(); 293 xc_state_t *xc = &xc_low_pri; 294 void *arg1, *arg2; 295 xcfunc_t func; 296 297 mutex_enter(&xc->xc_lock); 298 for (;;) { 299 while (!ci->ci_data.cpu_xcall_pending) { 300 if (xc->xc_headp == xc->xc_donep) { 301 cv_broadcast(&xc->xc_busy); 302 } 303 cv_wait(&ci->ci_data.cpu_xcall, &xc->xc_lock); 304 KASSERT(ci == curcpu()); 305 } 306 ci->ci_data.cpu_xcall_pending = false; 307 func = xc->xc_func; 308 arg1 = xc->xc_arg1; 309 arg2 = xc->xc_arg2; 310 mutex_exit(&xc->xc_lock); 311 312 KASSERT(func != NULL); 313 (*func)(arg1, arg2); 314 315 mutex_enter(&xc->xc_lock); 316 xc->xc_donep++; 317 } 318 /* NOTREACHED */ 319 } 320 321 /* 322 * xc_ipi_handler: 323 * 324 * Handler of cross-call IPI. 325 */ 326 void 327 xc_ipi_handler(void) 328 { 329 /* Executes xc_highpri_intr() via software interrupt. */ 330 softint_schedule(xc_sih); 331 } 332 333 /* 334 * xc_highpri_intr: 335 * 336 * A software interrupt handler for high priority calls. 337 */ 338 static void 339 xc_highpri_intr(void *dummy) 340 { 341 xc_state_t *xc = &xc_high_pri; 342 void *arg1, *arg2; 343 xcfunc_t func; 344 345 KASSERT(cpu_softintr_p()); 346 /* 347 * Lock-less fetch of function and its arguments. 348 * Safe since it cannot change at this point. 349 */ 350 KASSERT(xc->xc_donep < xc->xc_headp); 351 func = xc->xc_func; 352 arg1 = xc->xc_arg1; 353 arg2 = xc->xc_arg2; 354 355 KASSERT(func != NULL); 356 (*func)(arg1, arg2); 357 358 /* 359 * Note the request as done, and if we have reached the head, 360 * cross-call has been processed - notify waiters, if any. 361 */ 362 mutex_enter(&xc->xc_lock); 363 if (++xc->xc_donep == xc->xc_headp) { 364 cv_broadcast(&xc->xc_busy); 365 } 366 mutex_exit(&xc->xc_lock); 367 } 368 369 /* 370 * xc_highpri: 371 * 372 * Trigger a high priority call on one or more CPUs. 373 */ 374 static inline uint64_t 375 xc_highpri(xcfunc_t func, void *arg1, void *arg2, struct cpu_info *ci) 376 { 377 xc_state_t *xc = &xc_high_pri; 378 uint64_t where; 379 380 mutex_enter(&xc->xc_lock); 381 while (xc->xc_headp != xc->xc_donep) { 382 cv_wait(&xc->xc_busy, &xc->xc_lock); 383 } 384 xc->xc_func = func; 385 xc->xc_arg1 = arg1; 386 xc->xc_arg2 = arg2; 387 xc->xc_headp += (ci ? 1 : ncpu); 388 where = xc->xc_headp; 389 mutex_exit(&xc->xc_lock); 390 391 /* 392 * Send the IPI once lock is released. 393 * Note: it will handle the local CPU case. 394 */ 395 396 #ifdef MULTIPROCESSOR 397 kpreempt_disable(); 398 if (curcpu() == ci) { 399 /* Unicast: local CPU. */ 400 xc_ipi_handler(); 401 } else if (ci) { 402 /* Unicast: remote CPU. */ 403 xc_send_ipi(ci); 404 } else { 405 /* Broadcast: all, including local. */ 406 xc_send_ipi(NULL); 407 xc_ipi_handler(); 408 } 409 kpreempt_enable(); 410 #else 411 KASSERT(ci == NULL || curcpu() == ci); 412 xc_ipi_handler(); 413 #endif 414 415 /* Indicate a high priority ticket. */ 416 return (where | XC_PRI_BIT); 417 } 418
Indexes created Tue Sep 30 20:09:53 GMT 2025