subr_xcall.c revision 1.5.2.3
1 1.5.2.3 matt /* subr_xcall.c,v 1.5.2.2 2007/11/06 23:32:21 matt Exp */ 2 1.5.2.2 matt 3 1.5.2.2 matt /*- 4 1.5.2.2 matt * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 1.5.2.2 matt * All rights reserved. 6 1.5.2.2 matt * 7 1.5.2.2 matt * This code is derived from software contributed to The NetBSD Foundation 8 1.5.2.2 matt * by Andrew Doran. 9 1.5.2.2 matt * 10 1.5.2.2 matt * Redistribution and use in source and binary forms, with or without 11 1.5.2.2 matt * modification, are permitted provided that the following conditions 12 1.5.2.2 matt * are met: 13 1.5.2.2 matt * 1. Redistributions of source code must retain the above copyright 14 1.5.2.2 matt * notice, this list of conditions and the following disclaimer. 15 1.5.2.2 matt * 2. Redistributions in binary form must reproduce the above copyright 16 1.5.2.2 matt * notice, this list of conditions and the following disclaimer in the 17 1.5.2.2 matt * documentation and/or other materials provided with the distribution. 18 1.5.2.2 matt * 3. All advertising materials mentioning features or use of this software 19 1.5.2.2 matt * must display the following acknowledgement: 20 1.5.2.2 matt * This product includes software developed by the NetBSD 21 1.5.2.2 matt * Foundation, Inc. and its contributors. 22 1.5.2.2 matt * 4. Neither the name of The NetBSD Foundation nor the names of its 23 1.5.2.2 matt * contributors may be used to endorse or promote products derived 24 1.5.2.2 matt * from this software without specific prior written permission. 25 1.5.2.2 matt * 26 1.5.2.2 matt * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 1.5.2.2 matt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 1.5.2.2 matt * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 1.5.2.2 matt * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 1.5.2.2 matt * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 1.5.2.2 matt * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 1.5.2.2 matt * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 1.5.2.2 matt * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 1.5.2.2 matt * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 1.5.2.2 matt * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 1.5.2.2 matt * POSSIBILITY OF SUCH DAMAGE. 37 1.5.2.2 matt */ 38 1.5.2.2 matt 39 1.5.2.2 matt /* 40 1.5.2.2 matt * Cross call support 41 1.5.2.2 matt * 42 1.5.2.2 matt * Background 43 1.5.2.2 matt * 44 1.5.2.2 matt * Sometimes it is necessary to modify hardware state that is tied 45 1.5.2.2 matt * directly to individual CPUs (such as a CPU's local timer), and 46 1.5.2.2 matt * these updates can not be done remotely by another CPU. The LWP 47 1.5.2.2 matt * requesting the update may be unable to guarantee that it will be 48 1.5.2.2 matt * running on the CPU where the update must occur, when the update 49 1.5.2.2 matt * occurs. 50 1.5.2.2 matt * 51 1.5.2.2 matt * Additionally, it's sometimes necessary to modify per-CPU software 52 1.5.2.2 matt * state from a remote CPU. Where these update operations are so 53 1.5.2.2 matt * rare or the access to the per-CPU data so frequent that the cost 54 1.5.2.2 matt * of using locking or atomic operations to provide coherency is 55 1.5.2.2 matt * prohibitive, another way must be found. 56 1.5.2.2 matt * 57 1.5.2.2 matt * Cross calls help to solve these types of problem by allowing 58 1.5.2.2 matt * any CPU in the system to request that an arbitrary function be 59 1.5.2.2 matt * executed on any other CPU. 60 1.5.2.2 matt * 61 1.5.2.2 matt * Implementation 62 1.5.2.2 matt * 63 1.5.2.2 matt * A slow mechanism for making 'low priority' cross calls is 64 1.5.2.2 matt * provided. The function to be executed runs on the remote CPU 65 1.5.2.2 matt * within a bound kthread. No queueing is provided, and the 66 1.5.2.2 matt * implementation uses global state. The function being called may 67 1.5.2.2 matt * block briefly on locks, but in doing so must be careful to not 68 1.5.2.2 matt * interfere with other cross calls in the system. The function is 69 1.5.2.2 matt * called with thread context and not from a soft interrupt, so it 70 1.5.2.2 matt * can ensure that it is not interrupting other code running on the 71 1.5.2.2 matt * CPU, and so has exclusive access to the CPU. Since this facility 72 1.5.2.2 matt * is heavyweight, it's expected that it will not be used often. 73 1.5.2.2 matt * 74 1.5.2.2 matt * Cross calls must not allocate memory, as the pagedaemon uses 75 1.5.2.2 matt * them (and memory allocation may need to wait on the pagedaemon). 76 1.5.2.2 matt * 77 1.5.2.2 matt * Future directions 78 1.5.2.2 matt * 79 1.5.2.2 matt * Add a low-overhead mechanism to run cross calls in interrupt 80 1.5.2.2 matt * context (XC_HIGHPRI). 81 1.5.2.2 matt */ 82 1.5.2.2 matt 83 1.5.2.2 matt #include <sys/cdefs.h> 84 1.5.2.3 matt __KERNEL_RCSID(0, "subr_xcall.c,v 1.5.2.2 2007/11/06 23:32:21 matt Exp"); 85 1.5.2.2 matt 86 1.5.2.2 matt #include <sys/types.h> 87 1.5.2.2 matt #include <sys/param.h> 88 1.5.2.2 matt #include <sys/xcall.h> 89 1.5.2.2 matt #include <sys/mutex.h> 90 1.5.2.2 matt #include <sys/condvar.h> 91 1.5.2.2 matt #include <sys/evcnt.h> 92 1.5.2.2 matt #include <sys/kthread.h> 93 1.5.2.2 matt #include <sys/cpu.h> 94 1.5.2.2 matt 95 1.5.2.2 matt static void xc_thread(void *); 96 1.5.2.2 matt static uint64_t xc_lowpri(u_int, xcfunc_t, void *, void *, struct cpu_info *); 97 1.5.2.2 matt 98 1.5.2.2 matt static kmutex_t xc_lock; 99 1.5.2.2 matt static xcfunc_t xc_func; 100 1.5.2.2 matt static void *xc_arg1; 101 1.5.2.2 matt static void *xc_arg2; 102 1.5.2.2 matt static kcondvar_t xc_busy; 103 1.5.2.2 matt static struct evcnt xc_unicast_ev; 104 1.5.2.2 matt static struct evcnt xc_broadcast_ev; 105 1.5.2.2 matt static uint64_t xc_headp; 106 1.5.2.2 matt static uint64_t xc_tailp; 107 1.5.2.2 matt static uint64_t xc_donep; 108 1.5.2.2 matt 109 1.5.2.2 matt /* 110 1.5.2.2 matt * xc_init_cpu: 111 1.5.2.2 matt * 112 1.5.2.2 matt * Initialize the cross-call subsystem. Called once for each CPU 113 1.5.2.2 matt * in the system as they are attached. 114 1.5.2.2 matt */ 115 1.5.2.2 matt void 116 1.5.2.2 matt xc_init_cpu(struct cpu_info *ci) 117 1.5.2.2 matt { 118 1.5.2.2 matt static bool again; 119 1.5.2.2 matt int error; 120 1.5.2.2 matt 121 1.5.2.2 matt if (!again) { 122 1.5.2.2 matt /* Autoconfiguration will prevent re-entry. */ 123 1.5.2.2 matt again = true; 124 1.5.2.2 matt mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_NONE); 125 1.5.2.2 matt cv_init(&xc_busy, "xcallbsy"); 126 1.5.2.2 matt evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 127 1.5.2.2 matt "crosscall", "unicast"); 128 1.5.2.2 matt evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 129 1.5.2.2 matt "crosscall", "broadcast"); 130 1.5.2.2 matt } 131 1.5.2.2 matt 132 1.5.2.2 matt cv_init(&ci->ci_data.cpu_xcall, "xcall"); 133 1.5.2.2 matt error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 134 1.5.2.3 matt NULL, NULL, "xcall/%u", ci->ci_index); 135 1.5.2.2 matt if (error != 0) 136 1.5.2.2 matt panic("xc_init_cpu: error %d", error); 137 1.5.2.2 matt } 138 1.5.2.2 matt 139 1.5.2.2 matt /* 140 1.5.2.2 matt * xc_unicast: 141 1.5.2.2 matt * 142 1.5.2.2 matt * Trigger a call on all CPUs in the system. 143 1.5.2.2 matt */ 144 1.5.2.2 matt uint64_t 145 1.5.2.2 matt xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) 146 1.5.2.2 matt { 147 1.5.2.2 matt 148 1.5.2.2 matt if ((flags & XC_HIGHPRI) != 0) { 149 1.5.2.2 matt panic("xc_unicast: no high priority crosscalls yet"); 150 1.5.2.2 matt } else { 151 1.5.2.2 matt return xc_lowpri(flags, func, arg1, arg2, NULL); 152 1.5.2.2 matt } 153 1.5.2.2 matt } 154 1.5.2.2 matt 155 1.5.2.2 matt /* 156 1.5.2.2 matt * xc_unicast: 157 1.5.2.2 matt * 158 1.5.2.2 matt * Trigger a call on one CPU. 159 1.5.2.2 matt */ 160 1.5.2.2 matt uint64_t 161 1.5.2.2 matt xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, 162 1.5.2.2 matt struct cpu_info *ci) 163 1.5.2.2 matt { 164 1.5.2.2 matt 165 1.5.2.2 matt if ((flags & XC_HIGHPRI) != 0) { 166 1.5.2.2 matt panic("xc_unicast: no high priority crosscalls yet"); 167 1.5.2.2 matt } else { 168 1.5.2.2 matt KASSERT(ci != NULL); 169 1.5.2.2 matt return xc_lowpri(flags, func, arg1, arg2, ci); 170 1.5.2.2 matt } 171 1.5.2.2 matt } 172 1.5.2.2 matt 173 1.5.2.2 matt /* 174 1.5.2.2 matt * xc_lowpri: 175 1.5.2.2 matt * 176 1.5.2.2 matt * Trigger a low priority call on one or more CPUs. 177 1.5.2.2 matt */ 178 1.5.2.2 matt static uint64_t 179 1.5.2.2 matt xc_lowpri(u_int flags, xcfunc_t func, void *arg1, void *arg2, 180 1.5.2.2 matt struct cpu_info *ci) 181 1.5.2.2 matt { 182 1.5.2.2 matt CPU_INFO_ITERATOR cii; 183 1.5.2.2 matt u_int where; 184 1.5.2.2 matt 185 1.5.2.2 matt mutex_enter(&xc_lock); 186 1.5.2.2 matt while (xc_headp != xc_tailp) 187 1.5.2.2 matt cv_wait(&xc_busy, &xc_lock); 188 1.5.2.2 matt xc_arg1 = arg1; 189 1.5.2.2 matt xc_arg2 = arg2; 190 1.5.2.2 matt xc_func = func; 191 1.5.2.2 matt if (ci == NULL) { 192 1.5.2.2 matt xc_broadcast_ev.ev_count++; 193 1.5.2.2 matt for (CPU_INFO_FOREACH(cii, ci)) { 194 1.5.2.2 matt xc_headp += 1; 195 1.5.2.2 matt ci->ci_data.cpu_xcall_pending = true; 196 1.5.2.2 matt cv_signal(&ci->ci_data.cpu_xcall); 197 1.5.2.2 matt } 198 1.5.2.2 matt } else { 199 1.5.2.2 matt xc_unicast_ev.ev_count++; 200 1.5.2.2 matt xc_headp += 1; 201 1.5.2.2 matt ci->ci_data.cpu_xcall_pending = true; 202 1.5.2.2 matt cv_signal(&ci->ci_data.cpu_xcall); 203 1.5.2.2 matt } 204 1.5.2.2 matt KASSERT(xc_tailp < xc_headp); 205 1.5.2.2 matt where = xc_headp; 206 1.5.2.2 matt mutex_exit(&xc_lock); 207 1.5.2.2 matt 208 1.5.2.2 matt return where; 209 1.5.2.2 matt } 210 1.5.2.2 matt 211 1.5.2.2 matt /* 212 1.5.2.2 matt * xc_wait: 213 1.5.2.2 matt * 214 1.5.2.2 matt * Wait for a cross call to complete. 215 1.5.2.2 matt */ 216 1.5.2.2 matt void 217 1.5.2.2 matt xc_wait(uint64_t where) 218 1.5.2.2 matt { 219 1.5.2.2 matt 220 1.5.2.2 matt if (xc_donep >= where) 221 1.5.2.2 matt return; 222 1.5.2.2 matt 223 1.5.2.2 matt mutex_enter(&xc_lock); 224 1.5.2.2 matt while (xc_donep < where) 225 1.5.2.2 matt cv_wait(&xc_busy, &xc_lock); 226 1.5.2.2 matt mutex_exit(&xc_lock); 227 1.5.2.2 matt } 228 1.5.2.2 matt 229 1.5.2.2 matt /* 230 1.5.2.2 matt * xc_thread: 231 1.5.2.2 matt * 232 1.5.2.2 matt * One thread per-CPU to dispatch low priority calls. 233 1.5.2.2 matt */ 234 1.5.2.2 matt static void 235 1.5.2.2 matt xc_thread(void *cookie) 236 1.5.2.2 matt { 237 1.5.2.2 matt void *arg1, *arg2; 238 1.5.2.2 matt struct cpu_info *ci; 239 1.5.2.2 matt xcfunc_t func; 240 1.5.2.2 matt 241 1.5.2.2 matt ci = curcpu(); 242 1.5.2.2 matt 243 1.5.2.2 matt mutex_enter(&xc_lock); 244 1.5.2.2 matt for (;;) { 245 1.5.2.2 matt while (!ci->ci_data.cpu_xcall_pending) { 246 1.5.2.2 matt if (xc_headp == xc_tailp) 247 1.5.2.2 matt cv_broadcast(&xc_busy); 248 1.5.2.2 matt cv_wait(&ci->ci_data.cpu_xcall, &xc_lock); 249 1.5.2.2 matt KASSERT(ci == curcpu()); 250 1.5.2.2 matt } 251 1.5.2.2 matt ci->ci_data.cpu_xcall_pending = false; 252 1.5.2.2 matt func = xc_func; 253 1.5.2.2 matt arg1 = xc_arg1; 254 1.5.2.2 matt arg2 = xc_arg2; 255 1.5.2.2 matt xc_tailp++; 256 1.5.2.2 matt mutex_exit(&xc_lock); 257 1.5.2.2 matt 258 1.5.2.2 matt (*func)(arg1, arg2); 259 1.5.2.2 matt 260 1.5.2.2 matt mutex_enter(&xc_lock); 261 1.5.2.2 matt xc_donep++; 262 1.5.2.2 matt } 263 1.5.2.2 matt /* NOTREACHED */ 264 1.5.2.2 matt } 265
Indexes created Tue Sep 30 20:09:53 GMT 2025