kern_cpu.c revision 1.57
1 /* $NetBSD: kern_cpu.c,v 1.57 2012/08/29 17:13:21 drochner Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.57 2012/08/29 17:13:21 drochner Exp $"); 60 61 #include "opt_cpu_ucode.h" 62 #include "opt_compat_netbsd.h" 63 64 #include <sys/param.h> 65 #include <sys/systm.h> 66 #include <sys/idle.h> 67 #include <sys/sched.h> 68 #include <sys/intr.h> 69 #include <sys/conf.h> 70 #include <sys/cpu.h> 71 #include <sys/cpuio.h> 72 #include <sys/proc.h> 73 #include <sys/percpu.h> 74 #include <sys/kernel.h> 75 #include <sys/kauth.h> 76 #include <sys/xcall.h> 77 #include <sys/pool.h> 78 #include <sys/kmem.h> 79 #include <sys/select.h> 80 #include <sys/namei.h> 81 #include <sys/callout.h> 82 83 #include <uvm/uvm_extern.h> 84 85 /* 86 * If the port has stated that cpu_data is the first thing in cpu_info, 87 * verify that the claim is true. This will prevent them from getting out 88 * of sync. 89 */ 90 #ifdef __HAVE_CPU_DATA_FIRST 91 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 92 #else 93 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 94 #endif 95 96 void cpuctlattach(int); 97 98 static void cpu_xc_online(struct cpu_info *); 99 static void cpu_xc_offline(struct cpu_info *); 100 101 dev_type_ioctl(cpuctl_ioctl); 102 103 const struct cdevsw cpuctl_cdevsw = { 104 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 105 nullstop, notty, nopoll, nommap, nokqfilter, 106 D_OTHER | D_MPSAFE 107 }; 108 109 kmutex_t cpu_lock __cacheline_aligned; 110 int ncpu __read_mostly; 111 int ncpuonline __read_mostly; 112 bool mp_online __read_mostly; 113 114 /* Note: set on mi_cpu_attach() and idle_loop(). */ 115 kcpuset_t * kcpuset_attached __read_mostly = NULL; 116 kcpuset_t * kcpuset_running __read_mostly = NULL; 117 118 struct cpuqueue cpu_queue __cacheline_aligned 119 = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 120 121 static struct cpu_info **cpu_infos __read_mostly; 122 123 /* 124 * mi_cpu_init: early initialisation of MI CPU related structures. 125 * 126 * Note: may not block and memory allocator is not yet available. 127 */ 128 void 129 mi_cpu_init(void) 130 { 131 132 mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE); 133 134 kcpuset_create(&kcpuset_attached, true); 135 kcpuset_create(&kcpuset_running, true); 136 kcpuset_set(kcpuset_running, 0); 137 } 138 139 int 140 mi_cpu_attach(struct cpu_info *ci) 141 { 142 int error; 143 144 KASSERT(maxcpus > 0); 145 146 ci->ci_index = ncpu; 147 kcpuset_set(kcpuset_attached, cpu_index(ci)); 148 149 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 150 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 151 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 152 153 /* This is useful for eg, per-cpu evcnt */ 154 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 155 cpu_index(ci)); 156 157 if (__predict_false(cpu_infos == NULL)) { 158 cpu_infos = 159 kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP); 160 } 161 cpu_infos[cpu_index(ci)] = ci; 162 163 sched_cpuattach(ci); 164 165 error = create_idle_lwp(ci); 166 if (error != 0) { 167 /* XXX revert sched_cpuattach */ 168 return error; 169 } 170 171 if (ci == curcpu()) 172 ci->ci_data.cpu_onproc = curlwp; 173 else 174 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 175 176 percpu_init_cpu(ci); 177 softint_init(ci); 178 callout_init_cpu(ci); 179 xc_init_cpu(ci); 180 pool_cache_cpu_init(ci); 181 selsysinit(ci); 182 cache_cpu_init(ci); 183 TAILQ_INIT(&ci->ci_data.cpu_biodone); 184 ncpu++; 185 ncpuonline++; 186 187 return 0; 188 } 189 190 void 191 cpuctlattach(int dummy) 192 { 193 194 KASSERT(cpu_infos != NULL); 195 } 196 197 int 198 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 199 { 200 CPU_INFO_ITERATOR cii; 201 cpustate_t *cs; 202 struct cpu_info *ci; 203 int error, i; 204 u_int id; 205 206 error = 0; 207 208 mutex_enter(&cpu_lock); 209 switch (cmd) { 210 case IOC_CPU_SETSTATE: 211 cs = data; 212 error = kauth_authorize_system(l->l_cred, 213 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 214 NULL); 215 if (error != 0) 216 break; 217 if (cs->cs_id >= maxcpus || 218 (ci = cpu_lookup(cs->cs_id)) == NULL) { 219 error = ESRCH; 220 break; 221 } 222 cpu_setintr(ci, cs->cs_intr); 223 error = cpu_setstate(ci, cs->cs_online); 224 break; 225 226 case IOC_CPU_GETSTATE: 227 cs = data; 228 id = cs->cs_id; 229 memset(cs, 0, sizeof(*cs)); 230 cs->cs_id = id; 231 if (cs->cs_id >= maxcpus || 232 (ci = cpu_lookup(id)) == NULL) { 233 error = ESRCH; 234 break; 235 } 236 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 237 cs->cs_online = false; 238 else 239 cs->cs_online = true; 240 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 241 cs->cs_intr = false; 242 else 243 cs->cs_intr = true; 244 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 245 cs->cs_lastmodhi = (int32_t) 246 (ci->ci_schedstate.spc_lastmod >> 32); 247 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 248 cs->cs_hwid = ci->ci_cpuid; 249 break; 250 251 case IOC_CPU_MAPID: 252 i = 0; 253 for (CPU_INFO_FOREACH(cii, ci)) { 254 if (i++ == *(int *)data) 255 break; 256 } 257 if (ci == NULL) 258 error = ESRCH; 259 else 260 *(int *)data = cpu_index(ci); 261 break; 262 263 case IOC_CPU_GETCOUNT: 264 *(int *)data = ncpu; 265 break; 266 267 #ifdef CPU_UCODE 268 case IOC_CPU_UCODE_GET_VERSION: 269 error = cpu_ucode_get_version((struct cpu_ucode_version *)data); 270 break; 271 272 /* XXX ifdef COMPAT */ 273 case OIOC_CPU_UCODE_GET_VERSION: 274 error = compat6_cpu_ucode_get_version((struct compat6_cpu_ucode *)data); 275 break; 276 277 case IOC_CPU_UCODE_APPLY: 278 error = kauth_authorize_machdep(l->l_cred, 279 KAUTH_MACHDEP_CPU_UCODE_APPLY, 280 NULL, NULL, NULL, NULL); 281 if (error != 0) 282 break; 283 error = cpu_ucode_apply((const struct cpu_ucode *)data); 284 break; 285 286 /* XXX ifdef COMPAT */ 287 case OIOC_CPU_UCODE_APPLY: 288 error = kauth_authorize_machdep(l->l_cred, 289 KAUTH_MACHDEP_CPU_UCODE_APPLY, 290 NULL, NULL, NULL, NULL); 291 if (error != 0) 292 break; 293 error = compat6_cpu_ucode_apply((const struct compat6_cpu_ucode *)data); 294 break; 295 #endif 296 297 default: 298 error = ENOTTY; 299 break; 300 } 301 mutex_exit(&cpu_lock); 302 303 return error; 304 } 305 306 struct cpu_info * 307 cpu_lookup(u_int idx) 308 { 309 struct cpu_info *ci; 310 311 KASSERT(idx < maxcpus); 312 313 if (__predict_false(cpu_infos == NULL)) { 314 KASSERT(idx == 0); 315 return curcpu(); 316 } 317 318 ci = cpu_infos[idx]; 319 KASSERT(ci == NULL || cpu_index(ci) == idx); 320 321 return ci; 322 } 323 324 static void 325 cpu_xc_offline(struct cpu_info *ci) 326 { 327 struct schedstate_percpu *spc, *mspc = NULL; 328 struct cpu_info *target_ci; 329 struct lwp *l; 330 CPU_INFO_ITERATOR cii; 331 int s; 332 333 /* 334 * Thread that made the cross call (separate context) holds 335 * cpu_lock on our behalf. 336 */ 337 spc = &ci->ci_schedstate; 338 s = splsched(); 339 spc->spc_flags |= SPCF_OFFLINE; 340 splx(s); 341 342 /* Take the first available CPU for the migration. */ 343 for (CPU_INFO_FOREACH(cii, target_ci)) { 344 mspc = &target_ci->ci_schedstate; 345 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 346 break; 347 } 348 KASSERT(target_ci != NULL); 349 350 /* 351 * Migrate all non-bound threads to the other CPU. Note that this 352 * runs from the xcall thread, thus handling of LSONPROC is not needed. 353 */ 354 mutex_enter(proc_lock); 355 LIST_FOREACH(l, &alllwp, l_list) { 356 struct cpu_info *mci; 357 358 lwp_lock(l); 359 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 360 lwp_unlock(l); 361 continue; 362 } 363 /* Regular case - no affinity. */ 364 if (l->l_affinity == NULL) { 365 lwp_migrate(l, target_ci); 366 continue; 367 } 368 /* Affinity is set, find an online CPU in the set. */ 369 for (CPU_INFO_FOREACH(cii, mci)) { 370 mspc = &mci->ci_schedstate; 371 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 372 kcpuset_isset(l->l_affinity, cpu_index(mci))) 373 break; 374 } 375 if (mci == NULL) { 376 lwp_unlock(l); 377 mutex_exit(proc_lock); 378 goto fail; 379 } 380 lwp_migrate(l, mci); 381 } 382 mutex_exit(proc_lock); 383 384 #ifdef __HAVE_MD_CPU_OFFLINE 385 cpu_offline_md(); 386 #endif 387 return; 388 fail: 389 /* Just unset the SPCF_OFFLINE flag, caller will check */ 390 s = splsched(); 391 spc->spc_flags &= ~SPCF_OFFLINE; 392 splx(s); 393 } 394 395 static void 396 cpu_xc_online(struct cpu_info *ci) 397 { 398 struct schedstate_percpu *spc; 399 int s; 400 401 spc = &ci->ci_schedstate; 402 s = splsched(); 403 spc->spc_flags &= ~SPCF_OFFLINE; 404 splx(s); 405 } 406 407 int 408 cpu_setstate(struct cpu_info *ci, bool online) 409 { 410 struct schedstate_percpu *spc; 411 CPU_INFO_ITERATOR cii; 412 struct cpu_info *ci2; 413 uint64_t where; 414 xcfunc_t func; 415 int nonline; 416 417 spc = &ci->ci_schedstate; 418 419 KASSERT(mutex_owned(&cpu_lock)); 420 421 if (online) { 422 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 423 return 0; 424 func = (xcfunc_t)cpu_xc_online; 425 ncpuonline++; 426 } else { 427 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 428 return 0; 429 nonline = 0; 430 /* 431 * Ensure that at least one CPU within the processor set 432 * stays online. Revisit this later. 433 */ 434 for (CPU_INFO_FOREACH(cii, ci2)) { 435 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 436 continue; 437 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 438 continue; 439 nonline++; 440 } 441 if (nonline == 1) 442 return EBUSY; 443 func = (xcfunc_t)cpu_xc_offline; 444 ncpuonline--; 445 } 446 447 where = xc_unicast(0, func, ci, NULL, ci); 448 xc_wait(where); 449 if (online) { 450 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 451 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 452 /* If was not set offline, then it is busy */ 453 return EBUSY; 454 } 455 456 spc->spc_lastmod = time_second; 457 return 0; 458 } 459 460 #ifdef __HAVE_INTR_CONTROL 461 static void 462 cpu_xc_intr(struct cpu_info *ci) 463 { 464 struct schedstate_percpu *spc; 465 int s; 466 467 spc = &ci->ci_schedstate; 468 s = splsched(); 469 spc->spc_flags &= ~SPCF_NOINTR; 470 splx(s); 471 } 472 473 static void 474 cpu_xc_nointr(struct cpu_info *ci) 475 { 476 struct schedstate_percpu *spc; 477 int s; 478 479 spc = &ci->ci_schedstate; 480 s = splsched(); 481 spc->spc_flags |= SPCF_NOINTR; 482 splx(s); 483 } 484 485 int 486 cpu_setintr(struct cpu_info *ci, bool intr) 487 { 488 struct schedstate_percpu *spc; 489 CPU_INFO_ITERATOR cii; 490 struct cpu_info *ci2; 491 uint64_t where; 492 xcfunc_t func; 493 int nintr; 494 495 spc = &ci->ci_schedstate; 496 497 KASSERT(mutex_owned(&cpu_lock)); 498 499 if (intr) { 500 if ((spc->spc_flags & SPCF_NOINTR) == 0) 501 return 0; 502 func = (xcfunc_t)cpu_xc_intr; 503 } else { 504 if ((spc->spc_flags & SPCF_NOINTR) != 0) 505 return 0; 506 /* 507 * Ensure that at least one CPU within the system 508 * is handing device interrupts. 509 */ 510 nintr = 0; 511 for (CPU_INFO_FOREACH(cii, ci2)) { 512 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 513 continue; 514 if (ci2 == ci) 515 continue; 516 nintr++; 517 } 518 if (nintr == 0) 519 return EBUSY; 520 func = (xcfunc_t)cpu_xc_nointr; 521 } 522 523 where = xc_unicast(0, func, ci, NULL, ci); 524 xc_wait(where); 525 if (intr) { 526 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 527 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 528 /* If was not set offline, then it is busy */ 529 return EBUSY; 530 } 531 532 /* Direct interrupts away from the CPU and record the change. */ 533 cpu_intr_redistribute(); 534 spc->spc_lastmod = time_second; 535 return 0; 536 } 537 #else /* __HAVE_INTR_CONTROL */ 538 int 539 cpu_setintr(struct cpu_info *ci, bool intr) 540 { 541 542 return EOPNOTSUPP; 543 } 544 545 u_int 546 cpu_intr_count(struct cpu_info *ci) 547 { 548 549 return 0; /* 0 == "don't know" */ 550 } 551 #endif /* __HAVE_INTR_CONTROL */ 552 553 bool 554 cpu_softintr_p(void) 555 { 556 557 return (curlwp->l_pflag & LP_INTR) != 0; 558 } 559 560 #ifdef CPU_UCODE 561 int 562 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 563 { 564 firmware_handle_t fwh; 565 int error; 566 567 if (sc->sc_blob != NULL) { 568 firmware_free(sc->sc_blob, 0); 569 sc->sc_blob = NULL; 570 sc->sc_blobsize = 0; 571 } 572 573 error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname); 574 if (error != 0) { 575 aprint_error("ucode: firmware_open failed: %i\n", error); 576 goto err0; 577 } 578 579 sc->sc_blobsize = firmware_get_size(fwh); 580 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 581 if (sc->sc_blob == NULL) { 582 error = ENOMEM; 583 firmware_close(fwh); 584 goto err0; 585 } 586 587 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 588 firmware_close(fwh); 589 if (error != 0) 590 goto err1; 591 592 return 0; 593 594 err1: 595 firmware_free(sc->sc_blob, 0); 596 sc->sc_blob = NULL; 597 sc->sc_blobsize = 0; 598 err0: 599 return error; 600 } 601 #endif 602
Indexes created Sun Sep 21 18:09:52 GMT 2025