kern_cpu.c revision 1.56
1 /* $NetBSD: kern_cpu.c,v 1.56 2012/06/13 23:00:05 joerg Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.56 2012/06/13 23:00:05 joerg Exp $"); 60 61 #include "opt_cpu_ucode.h" 62 63 #include <sys/param.h> 64 #include <sys/systm.h> 65 #include <sys/idle.h> 66 #include <sys/sched.h> 67 #include <sys/intr.h> 68 #include <sys/conf.h> 69 #include <sys/cpu.h> 70 #include <sys/cpuio.h> 71 #include <sys/proc.h> 72 #include <sys/percpu.h> 73 #include <sys/kernel.h> 74 #include <sys/kauth.h> 75 #include <sys/xcall.h> 76 #include <sys/pool.h> 77 #include <sys/kmem.h> 78 #include <sys/select.h> 79 #include <sys/namei.h> 80 #include <sys/callout.h> 81 82 #include <uvm/uvm_extern.h> 83 84 /* 85 * If the port has stated that cpu_data is the first thing in cpu_info, 86 * verify that the claim is true. This will prevent them from getting out 87 * of sync. 88 */ 89 #ifdef __HAVE_CPU_DATA_FIRST 90 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 91 #else 92 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 93 #endif 94 95 void cpuctlattach(int); 96 97 static void cpu_xc_online(struct cpu_info *); 98 static void cpu_xc_offline(struct cpu_info *); 99 100 dev_type_ioctl(cpuctl_ioctl); 101 102 const struct cdevsw cpuctl_cdevsw = { 103 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 104 nullstop, notty, nopoll, nommap, nokqfilter, 105 D_OTHER | D_MPSAFE 106 }; 107 108 kmutex_t cpu_lock __cacheline_aligned; 109 int ncpu __read_mostly; 110 int ncpuonline __read_mostly; 111 bool mp_online __read_mostly; 112 113 /* Note: set on mi_cpu_attach() and idle_loop(). */ 114 kcpuset_t * kcpuset_attached __read_mostly = NULL; 115 kcpuset_t * kcpuset_running __read_mostly = NULL; 116 117 struct cpuqueue cpu_queue __cacheline_aligned 118 = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 119 120 static struct cpu_info **cpu_infos __read_mostly; 121 122 /* 123 * mi_cpu_init: early initialisation of MI CPU related structures. 124 * 125 * Note: may not block and memory allocator is not yet available. 126 */ 127 void 128 mi_cpu_init(void) 129 { 130 131 mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE); 132 133 kcpuset_create(&kcpuset_attached, true); 134 kcpuset_create(&kcpuset_running, true); 135 kcpuset_set(kcpuset_running, 0); 136 } 137 138 int 139 mi_cpu_attach(struct cpu_info *ci) 140 { 141 int error; 142 143 KASSERT(maxcpus > 0); 144 145 ci->ci_index = ncpu; 146 kcpuset_set(kcpuset_attached, cpu_index(ci)); 147 148 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 149 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 150 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 151 152 /* This is useful for eg, per-cpu evcnt */ 153 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 154 cpu_index(ci)); 155 156 if (__predict_false(cpu_infos == NULL)) { 157 cpu_infos = 158 kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP); 159 } 160 cpu_infos[cpu_index(ci)] = ci; 161 162 sched_cpuattach(ci); 163 164 error = create_idle_lwp(ci); 165 if (error != 0) { 166 /* XXX revert sched_cpuattach */ 167 return error; 168 } 169 170 if (ci == curcpu()) 171 ci->ci_data.cpu_onproc = curlwp; 172 else 173 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 174 175 percpu_init_cpu(ci); 176 softint_init(ci); 177 callout_init_cpu(ci); 178 xc_init_cpu(ci); 179 pool_cache_cpu_init(ci); 180 selsysinit(ci); 181 cache_cpu_init(ci); 182 TAILQ_INIT(&ci->ci_data.cpu_biodone); 183 ncpu++; 184 ncpuonline++; 185 186 return 0; 187 } 188 189 void 190 cpuctlattach(int dummy) 191 { 192 193 KASSERT(cpu_infos != NULL); 194 } 195 196 int 197 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 198 { 199 CPU_INFO_ITERATOR cii; 200 cpustate_t *cs; 201 struct cpu_info *ci; 202 int error, i; 203 u_int id; 204 205 error = 0; 206 207 mutex_enter(&cpu_lock); 208 switch (cmd) { 209 case IOC_CPU_SETSTATE: 210 cs = data; 211 error = kauth_authorize_system(l->l_cred, 212 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 213 NULL); 214 if (error != 0) 215 break; 216 if (cs->cs_id >= maxcpus || 217 (ci = cpu_lookup(cs->cs_id)) == NULL) { 218 error = ESRCH; 219 break; 220 } 221 cpu_setintr(ci, cs->cs_intr); 222 error = cpu_setstate(ci, cs->cs_online); 223 break; 224 225 case IOC_CPU_GETSTATE: 226 cs = data; 227 id = cs->cs_id; 228 memset(cs, 0, sizeof(*cs)); 229 cs->cs_id = id; 230 if (cs->cs_id >= maxcpus || 231 (ci = cpu_lookup(id)) == NULL) { 232 error = ESRCH; 233 break; 234 } 235 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 236 cs->cs_online = false; 237 else 238 cs->cs_online = true; 239 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 240 cs->cs_intr = false; 241 else 242 cs->cs_intr = true; 243 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 244 cs->cs_lastmodhi = (int32_t) 245 (ci->ci_schedstate.spc_lastmod >> 32); 246 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 247 cs->cs_hwid = ci->ci_cpuid; 248 break; 249 250 case IOC_CPU_MAPID: 251 i = 0; 252 for (CPU_INFO_FOREACH(cii, ci)) { 253 if (i++ == *(int *)data) 254 break; 255 } 256 if (ci == NULL) 257 error = ESRCH; 258 else 259 *(int *)data = cpu_index(ci); 260 break; 261 262 case IOC_CPU_GETCOUNT: 263 *(int *)data = ncpu; 264 break; 265 266 #ifdef CPU_UCODE 267 case IOC_CPU_UCODE_GET_VERSION: 268 error = cpu_ucode_get_version(data); 269 break; 270 271 case IOC_CPU_UCODE_APPLY: 272 error = kauth_authorize_machdep(l->l_cred, 273 KAUTH_MACHDEP_CPU_UCODE_APPLY, 274 NULL, NULL, NULL, NULL); 275 if (error != 0) 276 break; 277 error = cpu_ucode_apply(data); 278 break; 279 #endif 280 281 default: 282 error = ENOTTY; 283 break; 284 } 285 mutex_exit(&cpu_lock); 286 287 return error; 288 } 289 290 struct cpu_info * 291 cpu_lookup(u_int idx) 292 { 293 struct cpu_info *ci; 294 295 KASSERT(idx < maxcpus); 296 297 if (__predict_false(cpu_infos == NULL)) { 298 KASSERT(idx == 0); 299 return curcpu(); 300 } 301 302 ci = cpu_infos[idx]; 303 KASSERT(ci == NULL || cpu_index(ci) == idx); 304 305 return ci; 306 } 307 308 static void 309 cpu_xc_offline(struct cpu_info *ci) 310 { 311 struct schedstate_percpu *spc, *mspc = NULL; 312 struct cpu_info *target_ci; 313 struct lwp *l; 314 CPU_INFO_ITERATOR cii; 315 int s; 316 317 /* 318 * Thread that made the cross call (separate context) holds 319 * cpu_lock on our behalf. 320 */ 321 spc = &ci->ci_schedstate; 322 s = splsched(); 323 spc->spc_flags |= SPCF_OFFLINE; 324 splx(s); 325 326 /* Take the first available CPU for the migration. */ 327 for (CPU_INFO_FOREACH(cii, target_ci)) { 328 mspc = &target_ci->ci_schedstate; 329 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 330 break; 331 } 332 KASSERT(target_ci != NULL); 333 334 /* 335 * Migrate all non-bound threads to the other CPU. Note that this 336 * runs from the xcall thread, thus handling of LSONPROC is not needed. 337 */ 338 mutex_enter(proc_lock); 339 LIST_FOREACH(l, &alllwp, l_list) { 340 struct cpu_info *mci; 341 342 lwp_lock(l); 343 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 344 lwp_unlock(l); 345 continue; 346 } 347 /* Regular case - no affinity. */ 348 if (l->l_affinity == NULL) { 349 lwp_migrate(l, target_ci); 350 continue; 351 } 352 /* Affinity is set, find an online CPU in the set. */ 353 for (CPU_INFO_FOREACH(cii, mci)) { 354 mspc = &mci->ci_schedstate; 355 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 356 kcpuset_isset(l->l_affinity, cpu_index(mci))) 357 break; 358 } 359 if (mci == NULL) { 360 lwp_unlock(l); 361 mutex_exit(proc_lock); 362 goto fail; 363 } 364 lwp_migrate(l, mci); 365 } 366 mutex_exit(proc_lock); 367 368 #ifdef __HAVE_MD_CPU_OFFLINE 369 cpu_offline_md(); 370 #endif 371 return; 372 fail: 373 /* Just unset the SPCF_OFFLINE flag, caller will check */ 374 s = splsched(); 375 spc->spc_flags &= ~SPCF_OFFLINE; 376 splx(s); 377 } 378 379 static void 380 cpu_xc_online(struct cpu_info *ci) 381 { 382 struct schedstate_percpu *spc; 383 int s; 384 385 spc = &ci->ci_schedstate; 386 s = splsched(); 387 spc->spc_flags &= ~SPCF_OFFLINE; 388 splx(s); 389 } 390 391 int 392 cpu_setstate(struct cpu_info *ci, bool online) 393 { 394 struct schedstate_percpu *spc; 395 CPU_INFO_ITERATOR cii; 396 struct cpu_info *ci2; 397 uint64_t where; 398 xcfunc_t func; 399 int nonline; 400 401 spc = &ci->ci_schedstate; 402 403 KASSERT(mutex_owned(&cpu_lock)); 404 405 if (online) { 406 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 407 return 0; 408 func = (xcfunc_t)cpu_xc_online; 409 ncpuonline++; 410 } else { 411 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 412 return 0; 413 nonline = 0; 414 /* 415 * Ensure that at least one CPU within the processor set 416 * stays online. Revisit this later. 417 */ 418 for (CPU_INFO_FOREACH(cii, ci2)) { 419 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 420 continue; 421 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 422 continue; 423 nonline++; 424 } 425 if (nonline == 1) 426 return EBUSY; 427 func = (xcfunc_t)cpu_xc_offline; 428 ncpuonline--; 429 } 430 431 where = xc_unicast(0, func, ci, NULL, ci); 432 xc_wait(where); 433 if (online) { 434 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 435 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 436 /* If was not set offline, then it is busy */ 437 return EBUSY; 438 } 439 440 spc->spc_lastmod = time_second; 441 return 0; 442 } 443 444 #ifdef __HAVE_INTR_CONTROL 445 static void 446 cpu_xc_intr(struct cpu_info *ci) 447 { 448 struct schedstate_percpu *spc; 449 int s; 450 451 spc = &ci->ci_schedstate; 452 s = splsched(); 453 spc->spc_flags &= ~SPCF_NOINTR; 454 splx(s); 455 } 456 457 static void 458 cpu_xc_nointr(struct cpu_info *ci) 459 { 460 struct schedstate_percpu *spc; 461 int s; 462 463 spc = &ci->ci_schedstate; 464 s = splsched(); 465 spc->spc_flags |= SPCF_NOINTR; 466 splx(s); 467 } 468 469 int 470 cpu_setintr(struct cpu_info *ci, bool intr) 471 { 472 struct schedstate_percpu *spc; 473 CPU_INFO_ITERATOR cii; 474 struct cpu_info *ci2; 475 uint64_t where; 476 xcfunc_t func; 477 int nintr; 478 479 spc = &ci->ci_schedstate; 480 481 KASSERT(mutex_owned(&cpu_lock)); 482 483 if (intr) { 484 if ((spc->spc_flags & SPCF_NOINTR) == 0) 485 return 0; 486 func = (xcfunc_t)cpu_xc_intr; 487 } else { 488 if ((spc->spc_flags & SPCF_NOINTR) != 0) 489 return 0; 490 /* 491 * Ensure that at least one CPU within the system 492 * is handing device interrupts. 493 */ 494 nintr = 0; 495 for (CPU_INFO_FOREACH(cii, ci2)) { 496 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 497 continue; 498 if (ci2 == ci) 499 continue; 500 nintr++; 501 } 502 if (nintr == 0) 503 return EBUSY; 504 func = (xcfunc_t)cpu_xc_nointr; 505 } 506 507 where = xc_unicast(0, func, ci, NULL, ci); 508 xc_wait(where); 509 if (intr) { 510 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 511 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 512 /* If was not set offline, then it is busy */ 513 return EBUSY; 514 } 515 516 /* Direct interrupts away from the CPU and record the change. */ 517 cpu_intr_redistribute(); 518 spc->spc_lastmod = time_second; 519 return 0; 520 } 521 #else /* __HAVE_INTR_CONTROL */ 522 int 523 cpu_setintr(struct cpu_info *ci, bool intr) 524 { 525 526 return EOPNOTSUPP; 527 } 528 529 u_int 530 cpu_intr_count(struct cpu_info *ci) 531 { 532 533 return 0; /* 0 == "don't know" */ 534 } 535 #endif /* __HAVE_INTR_CONTROL */ 536 537 bool 538 cpu_softintr_p(void) 539 { 540 541 return (curlwp->l_pflag & LP_INTR) != 0; 542 } 543 544 #ifdef CPU_UCODE 545 int 546 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 547 { 548 firmware_handle_t fwh; 549 int error; 550 551 if (sc->sc_blob != NULL) { 552 firmware_free(sc->sc_blob, 0); 553 sc->sc_blob = NULL; 554 sc->sc_blobsize = 0; 555 } 556 557 error = cpu_ucode_md_open(&fwh, fwname); 558 if (error != 0) { 559 aprint_error("ucode: firmware_open failed: %i\n", error); 560 goto err0; 561 } 562 563 sc->sc_blobsize = firmware_get_size(fwh); 564 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 565 if (sc->sc_blob == NULL) { 566 error = ENOMEM; 567 firmware_close(fwh); 568 goto err0; 569 } 570 571 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 572 firmware_close(fwh); 573 if (error != 0) 574 goto err1; 575 576 return 0; 577 578 err1: 579 firmware_free(sc->sc_blob, 0); 580 sc->sc_blob = NULL; 581 sc->sc_blobsize = 0; 582 err0: 583 return error; 584 } 585 #endif 586
Indexes created Sun Sep 21 20:09:37 GMT 2025