subr_xcall.c revision 1.4.2.2
1 1.4.2.2 yamt /* $NetBSD: subr_xcall.c,v 1.4.2.2 2007/10/27 11:35:34 yamt Exp $ */ 2 1.4.2.2 yamt 3 1.4.2.2 yamt /*- 4 1.4.2.2 yamt * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 1.4.2.2 yamt * All rights reserved. 6 1.4.2.2 yamt * 7 1.4.2.2 yamt * This code is derived from software contributed to The NetBSD Foundation 8 1.4.2.2 yamt * by Andrew Doran. 9 1.4.2.2 yamt * 10 1.4.2.2 yamt * Redistribution and use in source and binary forms, with or without 11 1.4.2.2 yamt * modification, are permitted provided that the following conditions 12 1.4.2.2 yamt * are met: 13 1.4.2.2 yamt * 1. Redistributions of source code must retain the above copyright 14 1.4.2.2 yamt * notice, this list of conditions and the following disclaimer. 15 1.4.2.2 yamt * 2. Redistributions in binary form must reproduce the above copyright 16 1.4.2.2 yamt * notice, this list of conditions and the following disclaimer in the 17 1.4.2.2 yamt * documentation and/or other materials provided with the distribution. 18 1.4.2.2 yamt * 3. All advertising materials mentioning features or use of this software 19 1.4.2.2 yamt * must display the following acknowledgement: 20 1.4.2.2 yamt * This product includes software developed by the NetBSD 21 1.4.2.2 yamt * Foundation, Inc. and its contributors. 22 1.4.2.2 yamt * 4. Neither the name of The NetBSD Foundation nor the names of its 23 1.4.2.2 yamt * contributors may be used to endorse or promote products derived 24 1.4.2.2 yamt * from this software without specific prior written permission. 25 1.4.2.2 yamt * 26 1.4.2.2 yamt * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 1.4.2.2 yamt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 1.4.2.2 yamt * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 1.4.2.2 yamt * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 1.4.2.2 yamt * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 1.4.2.2 yamt * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 1.4.2.2 yamt * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 1.4.2.2 yamt * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 1.4.2.2 yamt * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 1.4.2.2 yamt * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 1.4.2.2 yamt * POSSIBILITY OF SUCH DAMAGE. 37 1.4.2.2 yamt */ 38 1.4.2.2 yamt 39 1.4.2.2 yamt /* 40 1.4.2.2 yamt * Cross call support 41 1.4.2.2 yamt * 42 1.4.2.2 yamt * Background 43 1.4.2.2 yamt * 44 1.4.2.2 yamt * Sometimes it is necessary to modify hardware state that is tied 45 1.4.2.2 yamt * directly to individual CPUs (such as a CPU's local timer), and 46 1.4.2.2 yamt * these updates can not be done remotely by another CPU. The LWP 47 1.4.2.2 yamt * requesting the update may be unable to guarantee that it will be 48 1.4.2.2 yamt * running on the CPU where the update must occur, when the update 49 1.4.2.2 yamt * occurs. 50 1.4.2.2 yamt * 51 1.4.2.2 yamt * Additionally, it's sometimes necessary to modify per-CPU software 52 1.4.2.2 yamt * state from a remote CPU. Where these update operations are so 53 1.4.2.2 yamt * rare or the access to the per-CPU data so frequent that the cost 54 1.4.2.2 yamt * of using locking or atomic operations to provide coherency is 55 1.4.2.2 yamt * prohibitive, another way must be found. 56 1.4.2.2 yamt * 57 1.4.2.2 yamt * Cross calls help to solve these types of problem by allowing 58 1.4.2.2 yamt * any CPU in the system to request that an arbitrary function be 59 1.4.2.2 yamt * executed on any other CPU. 60 1.4.2.2 yamt * 61 1.4.2.2 yamt * Implementation 62 1.4.2.2 yamt * 63 1.4.2.2 yamt * A slow mechanism for making 'low priority' cross calls is 64 1.4.2.2 yamt * provided. The function to be executed runs on the remote CPU 65 1.4.2.2 yamt * within a bound kthread. No queueing is provided, and the 66 1.4.2.2 yamt * implementation uses global state. The function being called may 67 1.4.2.2 yamt * block briefly on locks, but in doing so must be careful to not 68 1.4.2.2 yamt * interfere with other cross calls in the system. The function is 69 1.4.2.2 yamt * called with thread context and not from a soft interrupt, so it 70 1.4.2.2 yamt * can ensure that it is not interrupting other code running on the 71 1.4.2.2 yamt * CPU, and so has exclusive access to the CPU. Since this facility 72 1.4.2.2 yamt * is heavyweight, it's expected that it will not be used often. 73 1.4.2.2 yamt * 74 1.4.2.2 yamt * Cross calls must not allocate memory, as the pagedaemon uses 75 1.4.2.2 yamt * them (and memory allocation may need to wait on the pagedaemon). 76 1.4.2.2 yamt * 77 1.4.2.2 yamt * Future directions 78 1.4.2.2 yamt * 79 1.4.2.2 yamt * Add a low-overhead mechanism to run cross calls in interrupt 80 1.4.2.2 yamt * context (XC_HIGHPRI). 81 1.4.2.2 yamt */ 82 1.4.2.2 yamt 83 1.4.2.2 yamt #include <sys/cdefs.h> 84 1.4.2.2 yamt __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.4.2.2 2007/10/27 11:35:34 yamt Exp $"); 85 1.4.2.2 yamt 86 1.4.2.2 yamt #include <sys/types.h> 87 1.4.2.2 yamt #include <sys/param.h> 88 1.4.2.2 yamt #include <sys/xcall.h> 89 1.4.2.2 yamt #include <sys/mutex.h> 90 1.4.2.2 yamt #include <sys/condvar.h> 91 1.4.2.2 yamt #include <sys/evcnt.h> 92 1.4.2.2 yamt #include <sys/kthread.h> 93 1.4.2.2 yamt #include <sys/cpu.h> 94 1.4.2.2 yamt 95 1.4.2.2 yamt #define PRI_XCALL 0 96 1.4.2.2 yamt 97 1.4.2.2 yamt static void xc_thread(void *); 98 1.4.2.2 yamt static uint64_t xc_lowpri(u_int, xcfunc_t, void *, void *, struct cpu_info *); 99 1.4.2.2 yamt 100 1.4.2.2 yamt static kmutex_t xc_lock; 101 1.4.2.2 yamt static xcfunc_t xc_func; 102 1.4.2.2 yamt static void *xc_arg1; 103 1.4.2.2 yamt static void *xc_arg2; 104 1.4.2.2 yamt static kcondvar_t xc_busy; 105 1.4.2.2 yamt static struct evcnt xc_unicast_ev; 106 1.4.2.2 yamt static struct evcnt xc_broadcast_ev; 107 1.4.2.2 yamt static uint64_t xc_headp; 108 1.4.2.2 yamt static uint64_t xc_tailp; 109 1.4.2.2 yamt static uint64_t xc_donep; 110 1.4.2.2 yamt 111 1.4.2.2 yamt /* 112 1.4.2.2 yamt * xc_init_cpu: 113 1.4.2.2 yamt * 114 1.4.2.2 yamt * Initialize the cross-call subsystem. Called once for each CPU 115 1.4.2.2 yamt * in the system as they are attached. 116 1.4.2.2 yamt */ 117 1.4.2.2 yamt void 118 1.4.2.2 yamt xc_init_cpu(struct cpu_info *ci) 119 1.4.2.2 yamt { 120 1.4.2.2 yamt static bool again; 121 1.4.2.2 yamt int error; 122 1.4.2.2 yamt 123 1.4.2.2 yamt if (!again) { 124 1.4.2.2 yamt /* Autoconfiguration will prevent re-entry. */ 125 1.4.2.2 yamt again = true; 126 1.4.2.2 yamt mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_NONE); 127 1.4.2.2 yamt cv_init(&xc_busy, "xcallbsy"); 128 1.4.2.2 yamt evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 129 1.4.2.2 yamt "crosscall", "unicast"); 130 1.4.2.2 yamt evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 131 1.4.2.2 yamt "crosscall", "broadcast"); 132 1.4.2.2 yamt } 133 1.4.2.2 yamt 134 1.4.2.2 yamt cv_init(&ci->ci_data.cpu_xcall, "xcall"); 135 1.4.2.2 yamt error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 136 1.4.2.2 yamt NULL, NULL, "xcall/%d", (int)ci->ci_cpuid); 137 1.4.2.2 yamt if (error != 0) 138 1.4.2.2 yamt panic("xc_init_cpu: error %d", error); 139 1.4.2.2 yamt } 140 1.4.2.2 yamt 141 1.4.2.2 yamt /* 142 1.4.2.2 yamt * xc_unicast: 143 1.4.2.2 yamt * 144 1.4.2.2 yamt * Trigger a call on all CPUs in the system. 145 1.4.2.2 yamt */ 146 1.4.2.2 yamt uint64_t 147 1.4.2.2 yamt xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) 148 1.4.2.2 yamt { 149 1.4.2.2 yamt 150 1.4.2.2 yamt if ((flags & XC_HIGHPRI) != 0) { 151 1.4.2.2 yamt panic("xc_unicast: no high priority crosscalls yet"); 152 1.4.2.2 yamt } else { 153 1.4.2.2 yamt return xc_lowpri(flags, func, arg1, arg2, NULL); 154 1.4.2.2 yamt } 155 1.4.2.2 yamt } 156 1.4.2.2 yamt 157 1.4.2.2 yamt /* 158 1.4.2.2 yamt * xc_unicast: 159 1.4.2.2 yamt * 160 1.4.2.2 yamt * Trigger a call on one CPU. 161 1.4.2.2 yamt */ 162 1.4.2.2 yamt uint64_t 163 1.4.2.2 yamt xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, 164 1.4.2.2 yamt struct cpu_info *ci) 165 1.4.2.2 yamt { 166 1.4.2.2 yamt 167 1.4.2.2 yamt if ((flags & XC_HIGHPRI) != 0) { 168 1.4.2.2 yamt panic("xc_unicast: no high priority crosscalls yet"); 169 1.4.2.2 yamt } else { 170 1.4.2.2 yamt KASSERT(ci != NULL); 171 1.4.2.2 yamt return xc_lowpri(flags, func, arg1, arg2, ci); 172 1.4.2.2 yamt } 173 1.4.2.2 yamt } 174 1.4.2.2 yamt 175 1.4.2.2 yamt /* 176 1.4.2.2 yamt * xc_lowpri: 177 1.4.2.2 yamt * 178 1.4.2.2 yamt * Trigger a low priority call on one or more CPUs. 179 1.4.2.2 yamt */ 180 1.4.2.2 yamt static uint64_t 181 1.4.2.2 yamt xc_lowpri(u_int flags, xcfunc_t func, void *arg1, void *arg2, 182 1.4.2.2 yamt struct cpu_info *ci) 183 1.4.2.2 yamt { 184 1.4.2.2 yamt CPU_INFO_ITERATOR cii; 185 1.4.2.2 yamt u_int where; 186 1.4.2.2 yamt 187 1.4.2.2 yamt mutex_enter(&xc_lock); 188 1.4.2.2 yamt while (xc_headp != xc_tailp) 189 1.4.2.2 yamt cv_wait(&xc_busy, &xc_lock); 190 1.4.2.2 yamt xc_arg1 = arg1; 191 1.4.2.2 yamt xc_arg2 = arg2; 192 1.4.2.2 yamt xc_func = func; 193 1.4.2.2 yamt if (ci == NULL) { 194 1.4.2.2 yamt xc_broadcast_ev.ev_count++; 195 1.4.2.2 yamt for (CPU_INFO_FOREACH(cii, ci)) { 196 1.4.2.2 yamt xc_headp += 1; 197 1.4.2.2 yamt ci->ci_data.cpu_xcall_pending = true; 198 1.4.2.2 yamt cv_signal(&ci->ci_data.cpu_xcall); 199 1.4.2.2 yamt } 200 1.4.2.2 yamt } else { 201 1.4.2.2 yamt xc_unicast_ev.ev_count++; 202 1.4.2.2 yamt xc_headp += 1; 203 1.4.2.2 yamt ci->ci_data.cpu_xcall_pending = true; 204 1.4.2.2 yamt cv_signal(&ci->ci_data.cpu_xcall); 205 1.4.2.2 yamt } 206 1.4.2.2 yamt KASSERT(xc_tailp < xc_headp); 207 1.4.2.2 yamt where = xc_headp; 208 1.4.2.2 yamt mutex_exit(&xc_lock); 209 1.4.2.2 yamt 210 1.4.2.2 yamt return where; 211 1.4.2.2 yamt } 212 1.4.2.2 yamt 213 1.4.2.2 yamt /* 214 1.4.2.2 yamt * xc_wait: 215 1.4.2.2 yamt * 216 1.4.2.2 yamt * Wait for a cross call to complete. 217 1.4.2.2 yamt */ 218 1.4.2.2 yamt void 219 1.4.2.2 yamt xc_wait(uint64_t where) 220 1.4.2.2 yamt { 221 1.4.2.2 yamt 222 1.4.2.2 yamt if (xc_donep >= where) 223 1.4.2.2 yamt return; 224 1.4.2.2 yamt 225 1.4.2.2 yamt mutex_enter(&xc_lock); 226 1.4.2.2 yamt while (xc_donep < where) 227 1.4.2.2 yamt cv_wait(&xc_busy, &xc_lock); 228 1.4.2.2 yamt mutex_exit(&xc_lock); 229 1.4.2.2 yamt } 230 1.4.2.2 yamt 231 1.4.2.2 yamt /* 232 1.4.2.2 yamt * xc_thread: 233 1.4.2.2 yamt * 234 1.4.2.2 yamt * One thread per-CPU to dispatch low priority calls. 235 1.4.2.2 yamt */ 236 1.4.2.2 yamt static void 237 1.4.2.2 yamt xc_thread(void *cookie) 238 1.4.2.2 yamt { 239 1.4.2.2 yamt void *arg1, *arg2; 240 1.4.2.2 yamt struct cpu_info *ci; 241 1.4.2.2 yamt xcfunc_t func; 242 1.4.2.2 yamt 243 1.4.2.2 yamt ci = curcpu(); 244 1.4.2.2 yamt 245 1.4.2.2 yamt mutex_enter(&xc_lock); 246 1.4.2.2 yamt for (;;) { 247 1.4.2.2 yamt while (!ci->ci_data.cpu_xcall_pending) { 248 1.4.2.2 yamt if (xc_headp == xc_tailp) 249 1.4.2.2 yamt cv_broadcast(&xc_busy); 250 1.4.2.2 yamt cv_wait(&ci->ci_data.cpu_xcall, &xc_lock); 251 1.4.2.2 yamt KASSERT(ci == curcpu()); 252 1.4.2.2 yamt } 253 1.4.2.2 yamt ci->ci_data.cpu_xcall_pending = false; 254 1.4.2.2 yamt func = xc_func; 255 1.4.2.2 yamt arg1 = xc_arg1; 256 1.4.2.2 yamt arg2 = xc_arg2; 257 1.4.2.2 yamt xc_tailp++; 258 1.4.2.2 yamt mutex_exit(&xc_lock); 259 1.4.2.2 yamt 260 1.4.2.2 yamt (*func)(arg1, arg2); 261 1.4.2.2 yamt 262 1.4.2.2 yamt mutex_enter(&xc_lock); 263 1.4.2.2 yamt xc_donep++; 264 1.4.2.2 yamt } 265 1.4.2.2 yamt /* NOTREACHED */ 266 1.4.2.2 yamt } 267
Indexes created Tue Sep 30 20:09:53 GMT 2025