subr_xcall.c revision 1.4.2.4
1 1.4.2.4 yamt /* $NetBSD: subr_xcall.c,v 1.4.2.4 2008/03/17 09:15:34 yamt Exp $ */ 2 1.4.2.2 yamt 3 1.4.2.2 yamt /*- 4 1.4.2.2 yamt * Copyright (c) 2007 The NetBSD Foundation, Inc. 5 1.4.2.2 yamt * All rights reserved. 6 1.4.2.2 yamt * 7 1.4.2.2 yamt * This code is derived from software contributed to The NetBSD Foundation 8 1.4.2.2 yamt * by Andrew Doran. 9 1.4.2.2 yamt * 10 1.4.2.2 yamt * Redistribution and use in source and binary forms, with or without 11 1.4.2.2 yamt * modification, are permitted provided that the following conditions 12 1.4.2.2 yamt * are met: 13 1.4.2.2 yamt * 1. Redistributions of source code must retain the above copyright 14 1.4.2.2 yamt * notice, this list of conditions and the following disclaimer. 15 1.4.2.2 yamt * 2. Redistributions in binary form must reproduce the above copyright 16 1.4.2.2 yamt * notice, this list of conditions and the following disclaimer in the 17 1.4.2.2 yamt * documentation and/or other materials provided with the distribution. 18 1.4.2.2 yamt * 3. All advertising materials mentioning features or use of this software 19 1.4.2.2 yamt * must display the following acknowledgement: 20 1.4.2.2 yamt * This product includes software developed by the NetBSD 21 1.4.2.2 yamt * Foundation, Inc. and its contributors. 22 1.4.2.2 yamt * 4. Neither the name of The NetBSD Foundation nor the names of its 23 1.4.2.2 yamt * contributors may be used to endorse or promote products derived 24 1.4.2.2 yamt * from this software without specific prior written permission. 25 1.4.2.2 yamt * 26 1.4.2.2 yamt * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 1.4.2.2 yamt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 1.4.2.2 yamt * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 1.4.2.2 yamt * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 1.4.2.2 yamt * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 1.4.2.2 yamt * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 1.4.2.2 yamt * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 1.4.2.2 yamt * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 1.4.2.2 yamt * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 1.4.2.2 yamt * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 1.4.2.2 yamt * POSSIBILITY OF SUCH DAMAGE. 37 1.4.2.2 yamt */ 38 1.4.2.2 yamt 39 1.4.2.2 yamt /* 40 1.4.2.2 yamt * Cross call support 41 1.4.2.2 yamt * 42 1.4.2.2 yamt * Background 43 1.4.2.2 yamt * 44 1.4.2.2 yamt * Sometimes it is necessary to modify hardware state that is tied 45 1.4.2.2 yamt * directly to individual CPUs (such as a CPU's local timer), and 46 1.4.2.2 yamt * these updates can not be done remotely by another CPU. The LWP 47 1.4.2.2 yamt * requesting the update may be unable to guarantee that it will be 48 1.4.2.2 yamt * running on the CPU where the update must occur, when the update 49 1.4.2.2 yamt * occurs. 50 1.4.2.2 yamt * 51 1.4.2.2 yamt * Additionally, it's sometimes necessary to modify per-CPU software 52 1.4.2.2 yamt * state from a remote CPU. Where these update operations are so 53 1.4.2.2 yamt * rare or the access to the per-CPU data so frequent that the cost 54 1.4.2.2 yamt * of using locking or atomic operations to provide coherency is 55 1.4.2.2 yamt * prohibitive, another way must be found. 56 1.4.2.2 yamt * 57 1.4.2.2 yamt * Cross calls help to solve these types of problem by allowing 58 1.4.2.2 yamt * any CPU in the system to request that an arbitrary function be 59 1.4.2.2 yamt * executed on any other CPU. 60 1.4.2.2 yamt * 61 1.4.2.2 yamt * Implementation 62 1.4.2.2 yamt * 63 1.4.2.2 yamt * A slow mechanism for making 'low priority' cross calls is 64 1.4.2.2 yamt * provided. The function to be executed runs on the remote CPU 65 1.4.2.2 yamt * within a bound kthread. No queueing is provided, and the 66 1.4.2.2 yamt * implementation uses global state. The function being called may 67 1.4.2.2 yamt * block briefly on locks, but in doing so must be careful to not 68 1.4.2.2 yamt * interfere with other cross calls in the system. The function is 69 1.4.2.2 yamt * called with thread context and not from a soft interrupt, so it 70 1.4.2.2 yamt * can ensure that it is not interrupting other code running on the 71 1.4.2.2 yamt * CPU, and so has exclusive access to the CPU. Since this facility 72 1.4.2.2 yamt * is heavyweight, it's expected that it will not be used often. 73 1.4.2.2 yamt * 74 1.4.2.2 yamt * Cross calls must not allocate memory, as the pagedaemon uses 75 1.4.2.2 yamt * them (and memory allocation may need to wait on the pagedaemon). 76 1.4.2.2 yamt * 77 1.4.2.2 yamt * Future directions 78 1.4.2.2 yamt * 79 1.4.2.2 yamt * Add a low-overhead mechanism to run cross calls in interrupt 80 1.4.2.2 yamt * context (XC_HIGHPRI). 81 1.4.2.2 yamt */ 82 1.4.2.2 yamt 83 1.4.2.2 yamt #include <sys/cdefs.h> 84 1.4.2.4 yamt __KERNEL_RCSID(0, "$NetBSD: subr_xcall.c,v 1.4.2.4 2008/03/17 09:15:34 yamt Exp $"); 85 1.4.2.2 yamt 86 1.4.2.2 yamt #include <sys/types.h> 87 1.4.2.2 yamt #include <sys/param.h> 88 1.4.2.2 yamt #include <sys/xcall.h> 89 1.4.2.2 yamt #include <sys/mutex.h> 90 1.4.2.2 yamt #include <sys/condvar.h> 91 1.4.2.2 yamt #include <sys/evcnt.h> 92 1.4.2.2 yamt #include <sys/kthread.h> 93 1.4.2.2 yamt #include <sys/cpu.h> 94 1.4.2.2 yamt 95 1.4.2.2 yamt static void xc_thread(void *); 96 1.4.2.2 yamt static uint64_t xc_lowpri(u_int, xcfunc_t, void *, void *, struct cpu_info *); 97 1.4.2.2 yamt 98 1.4.2.2 yamt static kmutex_t xc_lock; 99 1.4.2.2 yamt static xcfunc_t xc_func; 100 1.4.2.2 yamt static void *xc_arg1; 101 1.4.2.2 yamt static void *xc_arg2; 102 1.4.2.2 yamt static kcondvar_t xc_busy; 103 1.4.2.2 yamt static struct evcnt xc_unicast_ev; 104 1.4.2.2 yamt static struct evcnt xc_broadcast_ev; 105 1.4.2.2 yamt static uint64_t xc_headp; 106 1.4.2.2 yamt static uint64_t xc_tailp; 107 1.4.2.2 yamt static uint64_t xc_donep; 108 1.4.2.2 yamt 109 1.4.2.2 yamt /* 110 1.4.2.2 yamt * xc_init_cpu: 111 1.4.2.2 yamt * 112 1.4.2.2 yamt * Initialize the cross-call subsystem. Called once for each CPU 113 1.4.2.2 yamt * in the system as they are attached. 114 1.4.2.2 yamt */ 115 1.4.2.2 yamt void 116 1.4.2.2 yamt xc_init_cpu(struct cpu_info *ci) 117 1.4.2.2 yamt { 118 1.4.2.2 yamt static bool again; 119 1.4.2.2 yamt int error; 120 1.4.2.2 yamt 121 1.4.2.2 yamt if (!again) { 122 1.4.2.2 yamt /* Autoconfiguration will prevent re-entry. */ 123 1.4.2.2 yamt again = true; 124 1.4.2.2 yamt mutex_init(&xc_lock, MUTEX_DEFAULT, IPL_NONE); 125 1.4.2.2 yamt cv_init(&xc_busy, "xcallbsy"); 126 1.4.2.2 yamt evcnt_attach_dynamic(&xc_unicast_ev, EVCNT_TYPE_MISC, NULL, 127 1.4.2.2 yamt "crosscall", "unicast"); 128 1.4.2.2 yamt evcnt_attach_dynamic(&xc_broadcast_ev, EVCNT_TYPE_MISC, NULL, 129 1.4.2.2 yamt "crosscall", "broadcast"); 130 1.4.2.2 yamt } 131 1.4.2.2 yamt 132 1.4.2.2 yamt cv_init(&ci->ci_data.cpu_xcall, "xcall"); 133 1.4.2.2 yamt error = kthread_create(PRI_XCALL, KTHREAD_MPSAFE, ci, xc_thread, 134 1.4.2.4 yamt NULL, NULL, "xcall/%u", ci->ci_index); 135 1.4.2.2 yamt if (error != 0) 136 1.4.2.2 yamt panic("xc_init_cpu: error %d", error); 137 1.4.2.2 yamt } 138 1.4.2.2 yamt 139 1.4.2.2 yamt /* 140 1.4.2.2 yamt * xc_unicast: 141 1.4.2.2 yamt * 142 1.4.2.2 yamt * Trigger a call on all CPUs in the system. 143 1.4.2.2 yamt */ 144 1.4.2.2 yamt uint64_t 145 1.4.2.2 yamt xc_broadcast(u_int flags, xcfunc_t func, void *arg1, void *arg2) 146 1.4.2.2 yamt { 147 1.4.2.2 yamt 148 1.4.2.2 yamt if ((flags & XC_HIGHPRI) != 0) { 149 1.4.2.2 yamt panic("xc_unicast: no high priority crosscalls yet"); 150 1.4.2.2 yamt } else { 151 1.4.2.2 yamt return xc_lowpri(flags, func, arg1, arg2, NULL); 152 1.4.2.2 yamt } 153 1.4.2.2 yamt } 154 1.4.2.2 yamt 155 1.4.2.2 yamt /* 156 1.4.2.2 yamt * xc_unicast: 157 1.4.2.2 yamt * 158 1.4.2.2 yamt * Trigger a call on one CPU. 159 1.4.2.2 yamt */ 160 1.4.2.2 yamt uint64_t 161 1.4.2.2 yamt xc_unicast(u_int flags, xcfunc_t func, void *arg1, void *arg2, 162 1.4.2.2 yamt struct cpu_info *ci) 163 1.4.2.2 yamt { 164 1.4.2.2 yamt 165 1.4.2.2 yamt if ((flags & XC_HIGHPRI) != 0) { 166 1.4.2.2 yamt panic("xc_unicast: no high priority crosscalls yet"); 167 1.4.2.2 yamt } else { 168 1.4.2.2 yamt KASSERT(ci != NULL); 169 1.4.2.2 yamt return xc_lowpri(flags, func, arg1, arg2, ci); 170 1.4.2.2 yamt } 171 1.4.2.2 yamt } 172 1.4.2.2 yamt 173 1.4.2.2 yamt /* 174 1.4.2.2 yamt * xc_lowpri: 175 1.4.2.2 yamt * 176 1.4.2.2 yamt * Trigger a low priority call on one or more CPUs. 177 1.4.2.2 yamt */ 178 1.4.2.2 yamt static uint64_t 179 1.4.2.2 yamt xc_lowpri(u_int flags, xcfunc_t func, void *arg1, void *arg2, 180 1.4.2.2 yamt struct cpu_info *ci) 181 1.4.2.2 yamt { 182 1.4.2.2 yamt CPU_INFO_ITERATOR cii; 183 1.4.2.2 yamt u_int where; 184 1.4.2.2 yamt 185 1.4.2.2 yamt mutex_enter(&xc_lock); 186 1.4.2.2 yamt while (xc_headp != xc_tailp) 187 1.4.2.2 yamt cv_wait(&xc_busy, &xc_lock); 188 1.4.2.2 yamt xc_arg1 = arg1; 189 1.4.2.2 yamt xc_arg2 = arg2; 190 1.4.2.2 yamt xc_func = func; 191 1.4.2.2 yamt if (ci == NULL) { 192 1.4.2.2 yamt xc_broadcast_ev.ev_count++; 193 1.4.2.2 yamt for (CPU_INFO_FOREACH(cii, ci)) { 194 1.4.2.2 yamt xc_headp += 1; 195 1.4.2.2 yamt ci->ci_data.cpu_xcall_pending = true; 196 1.4.2.2 yamt cv_signal(&ci->ci_data.cpu_xcall); 197 1.4.2.2 yamt } 198 1.4.2.2 yamt } else { 199 1.4.2.2 yamt xc_unicast_ev.ev_count++; 200 1.4.2.2 yamt xc_headp += 1; 201 1.4.2.2 yamt ci->ci_data.cpu_xcall_pending = true; 202 1.4.2.2 yamt cv_signal(&ci->ci_data.cpu_xcall); 203 1.4.2.2 yamt } 204 1.4.2.2 yamt KASSERT(xc_tailp < xc_headp); 205 1.4.2.2 yamt where = xc_headp; 206 1.4.2.2 yamt mutex_exit(&xc_lock); 207 1.4.2.2 yamt 208 1.4.2.2 yamt return where; 209 1.4.2.2 yamt } 210 1.4.2.2 yamt 211 1.4.2.2 yamt /* 212 1.4.2.2 yamt * xc_wait: 213 1.4.2.2 yamt * 214 1.4.2.2 yamt * Wait for a cross call to complete. 215 1.4.2.2 yamt */ 216 1.4.2.2 yamt void 217 1.4.2.2 yamt xc_wait(uint64_t where) 218 1.4.2.2 yamt { 219 1.4.2.2 yamt 220 1.4.2.2 yamt if (xc_donep >= where) 221 1.4.2.2 yamt return; 222 1.4.2.2 yamt 223 1.4.2.2 yamt mutex_enter(&xc_lock); 224 1.4.2.2 yamt while (xc_donep < where) 225 1.4.2.2 yamt cv_wait(&xc_busy, &xc_lock); 226 1.4.2.2 yamt mutex_exit(&xc_lock); 227 1.4.2.2 yamt } 228 1.4.2.2 yamt 229 1.4.2.2 yamt /* 230 1.4.2.2 yamt * xc_thread: 231 1.4.2.2 yamt * 232 1.4.2.2 yamt * One thread per-CPU to dispatch low priority calls. 233 1.4.2.2 yamt */ 234 1.4.2.2 yamt static void 235 1.4.2.2 yamt xc_thread(void *cookie) 236 1.4.2.2 yamt { 237 1.4.2.2 yamt void *arg1, *arg2; 238 1.4.2.2 yamt struct cpu_info *ci; 239 1.4.2.2 yamt xcfunc_t func; 240 1.4.2.2 yamt 241 1.4.2.2 yamt ci = curcpu(); 242 1.4.2.2 yamt 243 1.4.2.2 yamt mutex_enter(&xc_lock); 244 1.4.2.2 yamt for (;;) { 245 1.4.2.2 yamt while (!ci->ci_data.cpu_xcall_pending) { 246 1.4.2.2 yamt if (xc_headp == xc_tailp) 247 1.4.2.2 yamt cv_broadcast(&xc_busy); 248 1.4.2.2 yamt cv_wait(&ci->ci_data.cpu_xcall, &xc_lock); 249 1.4.2.2 yamt KASSERT(ci == curcpu()); 250 1.4.2.2 yamt } 251 1.4.2.2 yamt ci->ci_data.cpu_xcall_pending = false; 252 1.4.2.2 yamt func = xc_func; 253 1.4.2.2 yamt arg1 = xc_arg1; 254 1.4.2.2 yamt arg2 = xc_arg2; 255 1.4.2.2 yamt xc_tailp++; 256 1.4.2.2 yamt mutex_exit(&xc_lock); 257 1.4.2.2 yamt 258 1.4.2.2 yamt (*func)(arg1, arg2); 259 1.4.2.2 yamt 260 1.4.2.2 yamt mutex_enter(&xc_lock); 261 1.4.2.2 yamt xc_donep++; 262 1.4.2.2 yamt } 263 1.4.2.2 yamt /* NOTREACHED */ 264 1.4.2.2 yamt } 265
Indexes created Tue Sep 30 20:09:53 GMT 2025