subr_pcu.c revision 1.3.2.2
1 1.3.2.2 bouyer /* $NetBSD: subr_pcu.c,v 1.3.2.2 2011/03/05 15:10:40 bouyer Exp $ */ 2 1.3.2.2 bouyer 3 1.3.2.2 bouyer /*- 4 1.3.2.2 bouyer * Copyright (c) 2011 The NetBSD Foundation, Inc. 5 1.3.2.2 bouyer * All rights reserved. 6 1.3.2.2 bouyer * 7 1.3.2.2 bouyer * This code is derived from software contributed to The NetBSD Foundation 8 1.3.2.2 bouyer * by Mindaugas Rasiukevicius. 9 1.3.2.2 bouyer * 10 1.3.2.2 bouyer * Redistribution and use in source and binary forms, with or without 11 1.3.2.2 bouyer * modification, are permitted provided that the following conditions 12 1.3.2.2 bouyer * are met: 13 1.3.2.2 bouyer * 1. Redistributions of source code must retain the above copyright 14 1.3.2.2 bouyer * notice, this list of conditions and the following disclaimer. 15 1.3.2.2 bouyer * 2. Redistributions in binary form must reproduce the above copyright 16 1.3.2.2 bouyer * notice, this list of conditions and the following disclaimer in the 17 1.3.2.2 bouyer * documentation and/or other materials provided with the distribution. 18 1.3.2.2 bouyer * 19 1.3.2.2 bouyer * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 1.3.2.2 bouyer * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 1.3.2.2 bouyer * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 1.3.2.2 bouyer * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 1.3.2.2 bouyer * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 1.3.2.2 bouyer * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 1.3.2.2 bouyer * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 1.3.2.2 bouyer * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 1.3.2.2 bouyer * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 1.3.2.2 bouyer * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 1.3.2.2 bouyer * POSSIBILITY OF SUCH DAMAGE. 30 1.3.2.2 bouyer */ 31 1.3.2.2 bouyer 32 1.3.2.2 bouyer /* 33 1.3.2.2 bouyer * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 1.3.2.2 bouyer * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 1.3.2.2 bouyer * 36 1.3.2.2 bouyer * Concurrency notes: 37 1.3.2.2 bouyer * 38 1.3.2.2 bouyer * PCU state may be loaded only by the current LWP, that is, curlwp. 39 1.3.2.2 bouyer * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 1.3.2.2 bouyer * 41 1.3.2.2 bouyer * Request for a PCU release can be from owner LWP (whether PCU state 42 1.3.2.2 bouyer * is on current CPU or remote CPU) or any other LWP running on that 43 1.3.2.2 bouyer * CPU (in such case, owner LWP is on a remote CPU or sleeping). 44 1.3.2.2 bouyer * 45 1.3.2.2 bouyer * In any case, PCU state can only be changed from the running CPU. 46 1.3.2.2 bouyer * If said PCU state is on the remote CPU, a cross-call will be sent 47 1.3.2.2 bouyer * by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 48 1.3.2.2 bouyer * may only be changed by current CPU, and lwp_t::l_pcu_cpu[id] may 49 1.3.2.2 bouyer * only be unset by the CPU which has PCU state loaded. 50 1.3.2.2 bouyer * 51 1.3.2.2 bouyer * There is a race condition: LWP may have a PCU state on a remote CPU, 52 1.3.2.2 bouyer * which it requests to be released via cross-call. At the same time, 53 1.3.2.2 bouyer * other LWP on remote CPU might release existing PCU state and load 54 1.3.2.2 bouyer * its own one. Cross-call may arrive after this and release different 55 1.3.2.2 bouyer * PCU state than intended. In such case, such LWP would re-load its 56 1.3.2.2 bouyer * PCU state again. 57 1.3.2.2 bouyer */ 58 1.3.2.2 bouyer 59 1.3.2.2 bouyer #include <sys/cdefs.h> 60 1.3.2.2 bouyer __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.3.2.2 2011/03/05 15:10:40 bouyer Exp $"); 61 1.3.2.2 bouyer 62 1.3.2.2 bouyer #include <sys/param.h> 63 1.3.2.2 bouyer #include <sys/cpu.h> 64 1.3.2.2 bouyer #include <sys/lwp.h> 65 1.3.2.2 bouyer #include <sys/pcu.h> 66 1.3.2.2 bouyer #include <sys/xcall.h> 67 1.3.2.2 bouyer 68 1.3.2.2 bouyer #if PCU_UNIT_COUNT > 0 69 1.3.2.2 bouyer 70 1.3.2.2 bouyer #define PCU_SAVE 0x01 /* Save PCU state to the LWP. */ 71 1.3.2.2 bouyer #define PCU_RELEASE 0x02 /* Release PCU state on the CPU. */ 72 1.3.2.2 bouyer 73 1.3.2.2 bouyer #if 0 74 1.3.2.2 bouyer /* 75 1.3.2.2 bouyer * pcu_init_lwp: initialize PCU structures for LWP. 76 1.3.2.2 bouyer */ 77 1.3.2.2 bouyer void 78 1.3.2.2 bouyer pcu_init_lwp(lwp_t *l) 79 1.3.2.2 bouyer { 80 1.3.2.2 bouyer 81 1.3.2.2 bouyer memset(l->l_pcu_cpu, 0, sizeof(uint32_t) * PCU_UNIT_COUNT); 82 1.3.2.2 bouyer l->l_pcu_used = 0; 83 1.3.2.2 bouyer } 84 1.3.2.2 bouyer #endif 85 1.3.2.2 bouyer 86 1.3.2.2 bouyer /* 87 1.3.2.2 bouyer * pcu_cpu_op: save/release PCU state on the current CPU. 88 1.3.2.2 bouyer * 89 1.3.2.2 bouyer * => Must be called at IPL_SOFTCLOCK or from the soft-interrupt. 90 1.3.2.2 bouyer */ 91 1.3.2.2 bouyer static void 92 1.3.2.2 bouyer pcu_cpu_op(const pcu_ops_t *pcu, const int flags) 93 1.3.2.2 bouyer { 94 1.3.2.2 bouyer const u_int id = pcu->pcu_id; 95 1.3.2.2 bouyer struct cpu_info *ci = curcpu(); 96 1.3.2.2 bouyer lwp_t *l = ci->ci_pcu_curlwp[id]; 97 1.3.2.2 bouyer 98 1.3.2.2 bouyer /* If no state - nothing to do. */ 99 1.3.2.2 bouyer if (l == NULL) { 100 1.3.2.2 bouyer return; 101 1.3.2.2 bouyer } 102 1.3.2.2 bouyer if (flags & PCU_SAVE) { 103 1.3.2.2 bouyer pcu->pcu_state_save(l, (flags & PCU_RELEASE) != 0); 104 1.3.2.2 bouyer } 105 1.3.2.2 bouyer if (flags & PCU_RELEASE) { 106 1.3.2.2 bouyer ci->ci_pcu_curlwp[id] = NULL; 107 1.3.2.2 bouyer l->l_pcu_cpu[id] = NULL; 108 1.3.2.2 bouyer } 109 1.3.2.2 bouyer } 110 1.3.2.2 bouyer 111 1.3.2.2 bouyer /* 112 1.3.2.2 bouyer * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 113 1.3.2.2 bouyer */ 114 1.3.2.2 bouyer static void 115 1.3.2.2 bouyer pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, int flags) 116 1.3.2.2 bouyer { 117 1.3.2.2 bouyer const u_int id = pcu->pcu_id; 118 1.3.2.2 bouyer struct cpu_info *ci; 119 1.3.2.2 bouyer uint64_t where; 120 1.3.2.2 bouyer int s; 121 1.3.2.2 bouyer 122 1.3.2.2 bouyer /* 123 1.3.2.2 bouyer * Caller should have re-checked if there is any state to manage. 124 1.3.2.2 bouyer * Block the interrupts and inspect again, since cross-call sent 125 1.3.2.2 bouyer * by remote CPU could have changed the state. 126 1.3.2.2 bouyer */ 127 1.3.2.2 bouyer s = splsoftclock(); 128 1.3.2.2 bouyer ci = l->l_pcu_cpu[id]; 129 1.3.2.2 bouyer if (ci == curcpu()) { 130 1.3.2.2 bouyer /* 131 1.3.2.2 bouyer * State is on the current CPU - just perform the operations. 132 1.3.2.2 bouyer */ 133 1.3.2.2 bouyer KASSERT(ci->ci_pcu_curlwp[id] == l); 134 1.3.2.2 bouyer 135 1.3.2.2 bouyer if (flags & PCU_SAVE) { 136 1.3.2.2 bouyer pcu->pcu_state_save(l, (flags & PCU_RELEASE) != 0); 137 1.3.2.2 bouyer } 138 1.3.2.2 bouyer if (flags & PCU_RELEASE) { 139 1.3.2.2 bouyer ci->ci_pcu_curlwp[id] = NULL; 140 1.3.2.2 bouyer l->l_pcu_cpu[id] = NULL; 141 1.3.2.2 bouyer } 142 1.3.2.2 bouyer splx(s); 143 1.3.2.2 bouyer return; 144 1.3.2.2 bouyer } 145 1.3.2.2 bouyer splx(s); 146 1.3.2.2 bouyer 147 1.3.2.2 bouyer if (__predict_false(ci == NULL)) { 148 1.3.2.2 bouyer /* Cross-call has won the race - no state to manage. */ 149 1.3.2.2 bouyer return; 150 1.3.2.2 bouyer } 151 1.3.2.2 bouyer 152 1.3.2.2 bouyer /* 153 1.3.2.2 bouyer * State is on the remote CPU - perform the operations there. 154 1.3.2.2 bouyer * Note: there is a race condition; see description in the top. 155 1.3.2.2 bouyer */ 156 1.3.2.2 bouyer where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 157 1.3.2.2 bouyer __UNCONST(pcu), (void *)(uintptr_t)flags, ci); 158 1.3.2.2 bouyer xc_wait(where); 159 1.3.2.2 bouyer 160 1.3.2.2 bouyer KASSERT((flags & PCU_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 161 1.3.2.2 bouyer } 162 1.3.2.2 bouyer 163 1.3.2.2 bouyer /* 164 1.3.2.2 bouyer * pcu_load: load/initialize the PCU state of current LWP on current CPU. 165 1.3.2.2 bouyer */ 166 1.3.2.2 bouyer void 167 1.3.2.2 bouyer pcu_load(const pcu_ops_t *pcu) 168 1.3.2.2 bouyer { 169 1.3.2.2 bouyer const u_int id = pcu->pcu_id; 170 1.3.2.2 bouyer struct cpu_info *ci, *curci; 171 1.3.2.2 bouyer lwp_t *l = curlwp; 172 1.3.2.2 bouyer uint64_t where; 173 1.3.2.2 bouyer int s; 174 1.3.2.2 bouyer 175 1.3.2.2 bouyer KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 176 1.3.2.2 bouyer 177 1.3.2.2 bouyer s = splsoftclock(); 178 1.3.2.2 bouyer curci = curcpu(); 179 1.3.2.2 bouyer ci = l->l_pcu_cpu[id]; 180 1.3.2.2 bouyer 181 1.3.2.2 bouyer /* Does this CPU already have our PCU state loaded? */ 182 1.3.2.2 bouyer if (ci == curci) { 183 1.3.2.2 bouyer KASSERT(curci->ci_pcu_curlwp[id] == l); 184 1.3.2.2 bouyer splx(s); 185 1.3.2.2 bouyer return; 186 1.3.2.2 bouyer } 187 1.3.2.2 bouyer 188 1.3.2.2 bouyer /* If PCU state of this LWP is on the remote CPU - save it there. */ 189 1.3.2.2 bouyer if (ci) { 190 1.3.2.2 bouyer splx(s); 191 1.3.2.2 bouyer /* Note: there is a race; see description in the top. */ 192 1.3.2.2 bouyer where = xc_unicast(XC_HIGHPRI, (xcfunc_t)pcu_cpu_op, 193 1.3.2.2 bouyer __UNCONST(pcu), (void *)(PCU_SAVE | PCU_RELEASE), ci); 194 1.3.2.2 bouyer xc_wait(where); 195 1.3.2.2 bouyer 196 1.3.2.2 bouyer /* Enter IPL_SOFTCLOCK and re-fetch the current CPU. */ 197 1.3.2.2 bouyer s = splsoftclock(); 198 1.3.2.2 bouyer curci = curcpu(); 199 1.3.2.2 bouyer } 200 1.3.2.2 bouyer KASSERT(l->l_pcu_cpu[id] == NULL); 201 1.3.2.2 bouyer 202 1.3.2.2 bouyer /* Save the PCU state on the current CPU, if there is any. */ 203 1.3.2.2 bouyer pcu_cpu_op(pcu, PCU_SAVE | PCU_RELEASE); 204 1.3.2.2 bouyer KASSERT(curci->ci_pcu_curlwp[id] == NULL); 205 1.3.2.2 bouyer 206 1.3.2.2 bouyer /* 207 1.3.2.2 bouyer * Finally, load the state for this LWP on this CPU. Indicate to 208 1.3.2.2 bouyer * load function whether PCU was used before. Note the usage. 209 1.3.2.2 bouyer */ 210 1.3.2.2 bouyer pcu->pcu_state_load(l, ((1 << id) & l->l_pcu_used) != 0); 211 1.3.2.2 bouyer curci->ci_pcu_curlwp[id] = l; 212 1.3.2.2 bouyer l->l_pcu_cpu[id] = curci; 213 1.3.2.2 bouyer l->l_pcu_used |= (1 << id); 214 1.3.2.2 bouyer splx(s); 215 1.3.2.2 bouyer } 216 1.3.2.2 bouyer 217 1.3.2.2 bouyer /* 218 1.3.2.2 bouyer * pcu_discard: discard the PCU state of current LWP. 219 1.3.2.2 bouyer */ 220 1.3.2.2 bouyer void 221 1.3.2.2 bouyer pcu_discard(const pcu_ops_t *pcu) 222 1.3.2.2 bouyer { 223 1.3.2.2 bouyer const u_int id = pcu->pcu_id; 224 1.3.2.2 bouyer lwp_t *l = curlwp; 225 1.3.2.2 bouyer 226 1.3.2.2 bouyer KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 227 1.3.2.2 bouyer 228 1.3.2.2 bouyer if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 229 1.3.2.2 bouyer return; 230 1.3.2.2 bouyer } 231 1.3.2.2 bouyer pcu_lwp_op(pcu, l, PCU_RELEASE); 232 1.3.2.2 bouyer l->l_pcu_used &= ~(1 << id); 233 1.3.2.2 bouyer } 234 1.3.2.2 bouyer 235 1.3.2.2 bouyer /* 236 1.3.2.2 bouyer * pcu_save_lwp: save PCU state to the given LWP. 237 1.3.2.2 bouyer */ 238 1.3.2.2 bouyer void 239 1.3.2.2 bouyer pcu_save_lwp(const pcu_ops_t *pcu, lwp_t *l) 240 1.3.2.2 bouyer { 241 1.3.2.2 bouyer const u_int id = pcu->pcu_id; 242 1.3.2.2 bouyer 243 1.3.2.2 bouyer KASSERT(!cpu_intr_p() && !cpu_softintr_p()); 244 1.3.2.2 bouyer 245 1.3.2.2 bouyer if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 246 1.3.2.2 bouyer return; 247 1.3.2.2 bouyer } 248 1.3.2.2 bouyer pcu_lwp_op(pcu, l, PCU_SAVE | PCU_RELEASE); 249 1.3.2.2 bouyer } 250 1.3.2.2 bouyer 251 1.3.2.2 bouyer /* 252 1.3.2.2 bouyer * pcu_used: return true if PCU was used (pcu_load() case) by the LWP. 253 1.3.2.2 bouyer */ 254 1.3.2.2 bouyer bool 255 1.3.2.2 bouyer pcu_used(const pcu_ops_t *pcu, lwp_t *l) 256 1.3.2.2 bouyer { 257 1.3.2.2 bouyer const u_int id = pcu->pcu_id; 258 1.3.2.2 bouyer 259 1.3.2.2 bouyer return l->l_pcu_used & (1 << id); 260 1.3.2.2 bouyer } 261 1.3.2.2 bouyer 262 1.3.2.2 bouyer #endif /* PCU_UNIT_COUNT > 0 */ 263
Indexes created Tue Sep 30 20:09:53 GMT 2025