1 /* $NetBSD: subr_pcu.c,v 1.29 2026/07/17 02:17:12 thorpej Exp $ */ 2 3 /*- 4 * Copyright (c) 2011, 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Mindaugas Rasiukevicius. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Per CPU Unit (PCU) - is an interface to manage synchronization of any 34 * per CPU context (unit) tied with LWP context. Typical use: FPU state. 35 * 36 * Concurrency notes: 37 * 38 * PCU state may be loaded only by the current LWP, that is, curlwp. 39 * Therefore, only LWP itself can set a CPU for lwp_t::l_pcu_cpu[id]. 40 * 41 * There are some important rules about operation calls. The request 42 * for a PCU release can be from a) the owner LWP (regardless whether 43 * the PCU state is on the current CPU or remote CPU) b) any other LWP 44 * running on that CPU (in such case, the owner LWP is on a remote CPU 45 * or sleeping). 46 * 47 * In any case, the PCU state can *only* be changed from the current 48 * CPU. If said PCU state is on the remote CPU, a cross-call will be 49 * sent by the owner LWP. Therefore struct cpu_info::ci_pcu_curlwp[id] 50 * may only be changed by the current CPU and lwp_t::l_pcu_cpu[id] may 51 * only be cleared by the CPU which has the PCU state loaded. 52 */ 53 54 #include "opt_multiprocessor.h" 55 56 #include <sys/cdefs.h> 57 __KERNEL_RCSID(0, "$NetBSD: subr_pcu.c,v 1.29 2026/07/17 02:17:12 thorpej Exp $"); 58 59 #include <sys/param.h> 60 #include <sys/cpu.h> 61 #include <sys/lwp.h> 62 #include <sys/pcu.h> 63 #include <sys/ipi.h> 64 65 #if PCU_UNIT_COUNT > 0 66 67 #if !defined(MULTIPROCESSOR) && !defined(_RUMPKERNEL) 68 #define PCU_SLIM 69 #endif 70 71 static inline void pcu_do_op(const pcu_ops_t *, lwp_t * const, const int); 72 static void pcu_lwp_op(const pcu_ops_t *, lwp_t *, const int); 73 74 /* 75 * Internal PCU commands for the pcu_do_op() function. 76 */ 77 #define PCU_CMD_SAVE 0x01 /* save PCU state to the LWP */ 78 #define PCU_CMD_RELEASE 0x02 /* release PCU state on the CPU */ 79 80 #ifndef PCU_SLIM 81 /* 82 * Message structure for another CPU passed via ipi(9). 83 */ 84 typedef struct { 85 const pcu_ops_t *pcu; 86 lwp_t * owner; 87 const int flags; 88 } pcu_ipi_msg_t; 89 #endif /* ! PCU_SLIM */ 90 91 /* 92 * PCU IPIs run at IPL_HIGH (aka IPL_PCU in this code). 93 */ 94 #define splpcu splhigh 95 96 /* 97 * pcu_available_p: true if lwp is allowed to use PCU state. 98 */ 99 static inline bool __diagused 100 pcu_available_p(struct lwp *l) 101 { 102 103 /* XXX Not sure this is safe unless l is locked! */ 104 return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) != LW_SYSTEM; 105 } 106 107 /* 108 * pcu_switchpoint: release PCU state if the LWP is being run on another CPU. 109 * This routine is called on each context switch by by mi_switch(). 110 */ 111 void 112 pcu_switchpoint(lwp_t *l) 113 { 114 const uint32_t pcu_valid = l->l_pcu_valid; 115 int s; 116 117 KASSERTMSG(l == curlwp, "l %p != curlwp %p", l, curlwp); 118 119 if (__predict_true(pcu_valid == 0)) { 120 /* PCUs are not in use. */ 121 return; 122 } 123 s = splpcu(); 124 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 125 if ((pcu_valid & (1U << id)) == 0) { 126 continue; 127 } 128 struct cpu_info * const pcu_ci = l->l_pcu_cpu[id]; 129 if (pcu_ci == l->l_cpu) { 130 KASSERT(pcu_ci->ci_pcu_curlwp[id] == l); 131 continue; 132 } 133 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 134 pcu->pcu_state_release(l); 135 } 136 splx(s); 137 } 138 139 /* 140 * pcu_discard_all: discard PCU state of the given LWP. 141 * 142 * Used by exec and LWP exit. 143 */ 144 void 145 pcu_discard_all(lwp_t *l) 146 { 147 const uint32_t pcu_valid = l->l_pcu_valid; 148 149 /* 150 * The check for LSIDL here is to catch the case where the LWP exits 151 * due to an error in the LWP creation path before it ever runs. 152 */ 153 KASSERT(l == curlwp || l->l_stat == LSIDL || 154 (!pcu_available_p(l) && pcu_valid == 0)); 155 156 if (__predict_true(pcu_valid == 0)) { 157 /* PCUs are not in use. */ 158 return; 159 } 160 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 161 if ((pcu_valid & (1U << id)) == 0) { 162 continue; 163 } 164 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 165 continue; 166 } 167 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 168 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 169 } 170 l->l_pcu_valid = 0; 171 } 172 173 /* 174 * pcu_save_all: save PCU state of the given LWP so that eg. coredump can 175 * examine it. 176 */ 177 void 178 pcu_save_all(lwp_t *l) 179 { 180 const uint32_t pcu_valid = l->l_pcu_valid; 181 int flags = PCU_CMD_SAVE; 182 183 /* If LW_WCORE, we are also releasing the state. */ 184 if (__predict_false(l->l_flag & LW_WCORE)) { 185 flags |= PCU_CMD_RELEASE; 186 } 187 188 /* 189 * Normally we save for the current LWP, but sometimes we get called 190 * with a different LWP (forking a system LWP or doing a coredump of 191 * a process with multiple threads) and we need to deal with that. 192 */ 193 KASSERT(l == curlwp || ((!pcu_available_p(l) || 194 (curlwp->l_proc == l->l_proc && l->l_stat == LSSUSPENDED)) && 195 pcu_valid == 0)); 196 197 if (__predict_true(pcu_valid == 0)) { 198 /* PCUs are not in use. */ 199 return; 200 } 201 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 202 if ((pcu_valid & (1U << id)) == 0) { 203 continue; 204 } 205 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 206 continue; 207 } 208 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 209 pcu_lwp_op(pcu, l, flags); 210 } 211 } 212 213 /* 214 * pcu_do_op: save/release PCU state on the current CPU. 215 * 216 * => Must be called at IPL_PCU or from the interrupt. 217 */ 218 static inline void 219 pcu_do_op(const pcu_ops_t *pcu, lwp_t * const l, const int flags) 220 { 221 struct cpu_info * const ci = curcpu(); 222 const u_int id = pcu->pcu_id; 223 224 KASSERT(l->l_pcu_cpu[id] == ci); 225 226 if (flags & PCU_CMD_SAVE) { 227 pcu->pcu_state_save(l); 228 } 229 if (flags & PCU_CMD_RELEASE) { 230 pcu->pcu_state_release(l); 231 ci->ci_pcu_curlwp[id] = NULL; 232 l->l_pcu_cpu[id] = NULL; 233 } 234 } 235 236 #ifndef PCU_SLIM 237 /* 238 * pcu_cpu_ipi: helper routine to call pcu_do_op() via ipi(9). 239 */ 240 static void 241 pcu_cpu_ipi(void *arg) 242 { 243 const pcu_ipi_msg_t *pcu_msg = arg; 244 const pcu_ops_t *pcu = pcu_msg->pcu; 245 const u_int id = pcu->pcu_id; 246 lwp_t *l = pcu_msg->owner; 247 248 KASSERT(pcu_msg->owner != NULL); 249 250 if (curcpu()->ci_pcu_curlwp[id] != l) { 251 /* 252 * Different ownership: another LWP raced with us and 253 * perform save and release. There is nothing to do. 254 */ 255 KASSERT(l->l_pcu_cpu[id] == NULL); 256 return; 257 } 258 pcu_do_op(pcu, l, pcu_msg->flags); 259 } 260 #endif /* ! PCU_SLIM */ 261 262 /* 263 * pcu_lwp_op: perform PCU state save, release or both operations on LWP. 264 */ 265 static void 266 pcu_lwp_op(const pcu_ops_t *pcu, lwp_t *l, const int flags) 267 { 268 const u_int id = pcu->pcu_id; 269 struct cpu_info *ci; 270 int s; 271 272 /* 273 * Caller should have re-checked if there is any state to manage. 274 * Block the interrupts and inspect again, since cross-call sent 275 * by remote CPU could have changed the state. 276 */ 277 s = splpcu(); 278 ci = l->l_pcu_cpu[id]; 279 if (ci == curcpu()) { 280 /* 281 * State is on the current CPU - just perform the operations. 282 */ 283 KASSERTMSG(ci->ci_pcu_curlwp[id] == l, 284 "%s: cpu%u: pcu_curlwp[%u] (%p) != l (%p)", 285 __func__, cpu_index(ci), id, ci->ci_pcu_curlwp[id], l); 286 pcu_do_op(pcu, l, flags); 287 splx(s); 288 return; 289 } 290 if (__predict_false(ci == NULL)) { 291 /* Cross-call has won the race - no state to manage. */ 292 splx(s); 293 return; 294 } 295 #ifdef PCU_SLIM 296 panic("%s", __func__); 297 #else 298 /* 299 * The state is on the remote CPU: perform the operation(s) there. 300 */ 301 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, .flags = flags }; 302 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 303 ipi_unicast(&ipi_msg, ci); 304 splx(s); 305 306 /* Wait for completion. */ 307 ipi_wait(&ipi_msg); 308 309 KASSERT((flags & PCU_CMD_RELEASE) == 0 || l->l_pcu_cpu[id] == NULL); 310 #endif /* PCU_SLIM */ 311 } 312 313 /* 314 * pcu_load: load/initialize the PCU state of current LWP on current CPU. 315 */ 316 void 317 pcu_load(const pcu_ops_t *pcu) 318 { 319 lwp_t *oncpu_lwp, * const l = curlwp; 320 const u_int id = pcu->pcu_id; 321 struct cpu_info *ci, *curci; 322 int s; 323 324 KASSERT(!cpu_intr_p()); 325 KASSERT(!cpu_softintr_p()); 326 327 s = splpcu(); 328 curci = curcpu(); 329 ci = l->l_pcu_cpu[id]; 330 331 /* Does this CPU already have our PCU state loaded? */ 332 if (ci == curci) { 333 /* 334 * Fault reoccurred while the PCU state is loaded and 335 * therefore PCU should be reenabled. This happens 336 * if LWP is context switched to another CPU and then 337 * switched back to the original CPU while the state 338 * on that CPU has not been changed by other LWPs. 339 * 340 * It may also happen due to instruction "bouncing" on 341 * some architectures. 342 */ 343 KASSERT(curci->ci_pcu_curlwp[id] == l); 344 KASSERT(pcu_valid_p(pcu, l)); 345 pcu->pcu_state_load(l, PCU_VALID | PCU_REENABLE); 346 splx(s); 347 return; 348 } 349 350 #ifdef PCU_SLIM 351 KASSERT(ci == NULL); 352 #else 353 /* If PCU state of this LWP is on the remote CPU - save it there. */ 354 if (ci) { 355 pcu_ipi_msg_t pcu_msg = { .pcu = pcu, .owner = l, 356 .flags = PCU_CMD_SAVE | PCU_CMD_RELEASE }; 357 ipi_msg_t ipi_msg = { .func = pcu_cpu_ipi, .arg = &pcu_msg }; 358 ipi_unicast(&ipi_msg, ci); 359 splx(s); 360 361 /* 362 * Wait for completion, re-enter IPL_PCU and re-fetch 363 * the current CPU. 364 */ 365 ipi_wait(&ipi_msg); 366 s = splpcu(); 367 curci = curcpu(); 368 } 369 #endif /* PCU_SLIM */ 370 KASSERT(l->l_pcu_cpu[id] == NULL); 371 372 /* Save the PCU state on the current CPU, if there is any. */ 373 if ((oncpu_lwp = curci->ci_pcu_curlwp[id]) != NULL) { 374 pcu_do_op(pcu, oncpu_lwp, PCU_CMD_SAVE | PCU_CMD_RELEASE); 375 KASSERT(curci->ci_pcu_curlwp[id] == NULL); 376 } 377 378 /* 379 * Finally, load the state for this LWP on this CPU. Indicate to 380 * the load function whether PCU state was valid before this call. 381 */ 382 const bool valid = ((1U << id) & l->l_pcu_valid) != 0; 383 pcu->pcu_state_load(l, valid ? PCU_VALID : 0); 384 curci->ci_pcu_curlwp[id] = l; 385 l->l_pcu_cpu[id] = curci; 386 l->l_pcu_valid |= (1U << id); 387 splx(s); 388 } 389 390 /* 391 * pcu_discard: discard the PCU state of the given LWP. If "valid" 392 * parameter is true, then keep considering the PCU state as valid. 393 */ 394 void 395 pcu_discard(const pcu_ops_t *pcu, lwp_t *l, bool valid) 396 { 397 const u_int id = pcu->pcu_id; 398 399 KASSERT(!cpu_intr_p()); 400 KASSERT(!cpu_softintr_p()); 401 402 if (__predict_false(valid)) { 403 l->l_pcu_valid |= (1U << id); 404 } else { 405 l->l_pcu_valid &= ~(1U << id); 406 } 407 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 408 return; 409 } 410 pcu_lwp_op(pcu, l, PCU_CMD_RELEASE); 411 } 412 413 /* 414 * pcu_save_lwp: save PCU state to the given LWP. 415 */ 416 void 417 pcu_save(const pcu_ops_t *pcu, lwp_t *l) 418 { 419 const u_int id = pcu->pcu_id; 420 421 KASSERT(!cpu_intr_p()); 422 KASSERT(!cpu_softintr_p()); 423 424 if (__predict_true(l->l_pcu_cpu[id] == NULL)) { 425 return; 426 } 427 pcu_lwp_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 428 } 429 430 /* 431 * pcu_save_all_on_cpu: save all PCU states on the current CPU. 432 */ 433 void 434 pcu_save_all_on_cpu(void) 435 { 436 int s; 437 438 s = splpcu(); 439 for (u_int id = 0; id < PCU_UNIT_COUNT; id++) { 440 const pcu_ops_t * const pcu = pcu_ops_md_defs[id]; 441 lwp_t *l; 442 443 if ((l = curcpu()->ci_pcu_curlwp[id]) != NULL) { 444 pcu_do_op(pcu, l, PCU_CMD_SAVE | PCU_CMD_RELEASE); 445 } 446 } 447 splx(s); 448 } 449 450 /* 451 * pcu_valid_p: return true if PCU state is considered valid. Generally, 452 * it always becomes "valid" when pcu_load() is called. 453 */ 454 bool 455 pcu_valid_p(const pcu_ops_t *pcu, const lwp_t *l) 456 { 457 const u_int id = pcu->pcu_id; 458 459 return (l->l_pcu_valid & (1U << id)) != 0; 460 } 461 462 #endif /* PCU_UNIT_COUNT > 0 */ 463
Indexes created Sun Aug 30 00:27:18 UTC 2026