kern_cpu.c revision 1.58
1 /* $NetBSD: kern_cpu.c,v 1.58 2012/09/01 00:24:43 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.58 2012/09/01 00:24:43 matt Exp $"); 60 61 #include "opt_cpu_ucode.h" 62 #include "opt_compat_netbsd.h" 63 64 #include <sys/param.h> 65 #include <sys/systm.h> 66 #include <sys/idle.h> 67 #include <sys/sched.h> 68 #include <sys/intr.h> 69 #include <sys/conf.h> 70 #include <sys/cpu.h> 71 #include <sys/cpuio.h> 72 #include <sys/proc.h> 73 #include <sys/percpu.h> 74 #include <sys/kernel.h> 75 #include <sys/kauth.h> 76 #include <sys/xcall.h> 77 #include <sys/pool.h> 78 #include <sys/kmem.h> 79 #include <sys/select.h> 80 #include <sys/namei.h> 81 #include <sys/callout.h> 82 83 #include <uvm/uvm_extern.h> 84 85 /* 86 * If the port has stated that cpu_data is the first thing in cpu_info, 87 * verify that the claim is true. This will prevent them from getting out 88 * of sync. 89 */ 90 #ifdef __HAVE_CPU_DATA_FIRST 91 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 92 #else 93 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 94 #endif 95 96 void cpuctlattach(int); 97 98 static void cpu_xc_online(struct cpu_info *); 99 static void cpu_xc_offline(struct cpu_info *); 100 101 dev_type_ioctl(cpuctl_ioctl); 102 103 const struct cdevsw cpuctl_cdevsw = { 104 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 105 nullstop, notty, nopoll, nommap, nokqfilter, 106 D_OTHER | D_MPSAFE 107 }; 108 109 kmutex_t cpu_lock __cacheline_aligned; 110 int ncpu __read_mostly; 111 int ncpuonline __read_mostly; 112 bool mp_online __read_mostly; 113 114 /* Note: set on mi_cpu_attach() and idle_loop(). */ 115 kcpuset_t * kcpuset_attached __read_mostly = NULL; 116 kcpuset_t * kcpuset_running __read_mostly = NULL; 117 118 struct cpuqueue cpu_queue __cacheline_aligned 119 = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 120 121 static struct cpu_info **cpu_infos __read_mostly; 122 123 /* 124 * mi_cpu_init: early initialisation of MI CPU related structures. 125 * 126 * Note: may not block and memory allocator is not yet available. 127 */ 128 void 129 mi_cpu_init(void) 130 { 131 132 mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE); 133 134 kcpuset_create(&kcpuset_attached, true); 135 kcpuset_create(&kcpuset_running, true); 136 kcpuset_set(kcpuset_running, 0); 137 } 138 139 int 140 mi_cpu_attach(struct cpu_info *ci) 141 { 142 int error; 143 144 KASSERT(maxcpus > 0); 145 146 ci->ci_index = ncpu; 147 kcpuset_set(kcpuset_attached, cpu_index(ci)); 148 149 /* 150 * Create a convenience cpuset of just ourselves. 151 */ 152 kcpuset_create(&ci->ci_data.cpu_kcpuset, true); 153 kcpuset_set(ci->ci_data.cpu_kcpuset, cpu_index(ci)); 154 155 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 156 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 157 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 158 159 /* This is useful for eg, per-cpu evcnt */ 160 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 161 cpu_index(ci)); 162 163 if (__predict_false(cpu_infos == NULL)) { 164 cpu_infos = 165 kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP); 166 } 167 cpu_infos[cpu_index(ci)] = ci; 168 169 sched_cpuattach(ci); 170 171 error = create_idle_lwp(ci); 172 if (error != 0) { 173 /* XXX revert sched_cpuattach */ 174 return error; 175 } 176 177 if (ci == curcpu()) 178 ci->ci_data.cpu_onproc = curlwp; 179 else 180 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 181 182 percpu_init_cpu(ci); 183 softint_init(ci); 184 callout_init_cpu(ci); 185 xc_init_cpu(ci); 186 pool_cache_cpu_init(ci); 187 selsysinit(ci); 188 cache_cpu_init(ci); 189 TAILQ_INIT(&ci->ci_data.cpu_biodone); 190 ncpu++; 191 ncpuonline++; 192 193 return 0; 194 } 195 196 void 197 cpuctlattach(int dummy) 198 { 199 200 KASSERT(cpu_infos != NULL); 201 } 202 203 int 204 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 205 { 206 CPU_INFO_ITERATOR cii; 207 cpustate_t *cs; 208 struct cpu_info *ci; 209 int error, i; 210 u_int id; 211 212 error = 0; 213 214 mutex_enter(&cpu_lock); 215 switch (cmd) { 216 case IOC_CPU_SETSTATE: 217 cs = data; 218 error = kauth_authorize_system(l->l_cred, 219 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 220 NULL); 221 if (error != 0) 222 break; 223 if (cs->cs_id >= maxcpus || 224 (ci = cpu_lookup(cs->cs_id)) == NULL) { 225 error = ESRCH; 226 break; 227 } 228 cpu_setintr(ci, cs->cs_intr); 229 error = cpu_setstate(ci, cs->cs_online); 230 break; 231 232 case IOC_CPU_GETSTATE: 233 cs = data; 234 id = cs->cs_id; 235 memset(cs, 0, sizeof(*cs)); 236 cs->cs_id = id; 237 if (cs->cs_id >= maxcpus || 238 (ci = cpu_lookup(id)) == NULL) { 239 error = ESRCH; 240 break; 241 } 242 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 243 cs->cs_online = false; 244 else 245 cs->cs_online = true; 246 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 247 cs->cs_intr = false; 248 else 249 cs->cs_intr = true; 250 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 251 cs->cs_lastmodhi = (int32_t) 252 (ci->ci_schedstate.spc_lastmod >> 32); 253 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 254 cs->cs_hwid = ci->ci_cpuid; 255 break; 256 257 case IOC_CPU_MAPID: 258 i = 0; 259 for (CPU_INFO_FOREACH(cii, ci)) { 260 if (i++ == *(int *)data) 261 break; 262 } 263 if (ci == NULL) 264 error = ESRCH; 265 else 266 *(int *)data = cpu_index(ci); 267 break; 268 269 case IOC_CPU_GETCOUNT: 270 *(int *)data = ncpu; 271 break; 272 273 #ifdef CPU_UCODE 274 case IOC_CPU_UCODE_GET_VERSION: 275 error = cpu_ucode_get_version((struct cpu_ucode_version *)data); 276 break; 277 278 /* XXX ifdef COMPAT */ 279 case OIOC_CPU_UCODE_GET_VERSION: 280 error = compat6_cpu_ucode_get_version((struct compat6_cpu_ucode *)data); 281 break; 282 283 case IOC_CPU_UCODE_APPLY: 284 error = kauth_authorize_machdep(l->l_cred, 285 KAUTH_MACHDEP_CPU_UCODE_APPLY, 286 NULL, NULL, NULL, NULL); 287 if (error != 0) 288 break; 289 error = cpu_ucode_apply((const struct cpu_ucode *)data); 290 break; 291 292 /* XXX ifdef COMPAT */ 293 case OIOC_CPU_UCODE_APPLY: 294 error = kauth_authorize_machdep(l->l_cred, 295 KAUTH_MACHDEP_CPU_UCODE_APPLY, 296 NULL, NULL, NULL, NULL); 297 if (error != 0) 298 break; 299 error = compat6_cpu_ucode_apply((const struct compat6_cpu_ucode *)data); 300 break; 301 #endif 302 303 default: 304 error = ENOTTY; 305 break; 306 } 307 mutex_exit(&cpu_lock); 308 309 return error; 310 } 311 312 struct cpu_info * 313 cpu_lookup(u_int idx) 314 { 315 struct cpu_info *ci; 316 317 KASSERT(idx < maxcpus); 318 319 if (__predict_false(cpu_infos == NULL)) { 320 KASSERT(idx == 0); 321 return curcpu(); 322 } 323 324 ci = cpu_infos[idx]; 325 KASSERT(ci == NULL || cpu_index(ci) == idx); 326 327 return ci; 328 } 329 330 static void 331 cpu_xc_offline(struct cpu_info *ci) 332 { 333 struct schedstate_percpu *spc, *mspc = NULL; 334 struct cpu_info *target_ci; 335 struct lwp *l; 336 CPU_INFO_ITERATOR cii; 337 int s; 338 339 /* 340 * Thread that made the cross call (separate context) holds 341 * cpu_lock on our behalf. 342 */ 343 spc = &ci->ci_schedstate; 344 s = splsched(); 345 spc->spc_flags |= SPCF_OFFLINE; 346 splx(s); 347 348 /* Take the first available CPU for the migration. */ 349 for (CPU_INFO_FOREACH(cii, target_ci)) { 350 mspc = &target_ci->ci_schedstate; 351 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 352 break; 353 } 354 KASSERT(target_ci != NULL); 355 356 /* 357 * Migrate all non-bound threads to the other CPU. Note that this 358 * runs from the xcall thread, thus handling of LSONPROC is not needed. 359 */ 360 mutex_enter(proc_lock); 361 LIST_FOREACH(l, &alllwp, l_list) { 362 struct cpu_info *mci; 363 364 lwp_lock(l); 365 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 366 lwp_unlock(l); 367 continue; 368 } 369 /* Regular case - no affinity. */ 370 if (l->l_affinity == NULL) { 371 lwp_migrate(l, target_ci); 372 continue; 373 } 374 /* Affinity is set, find an online CPU in the set. */ 375 for (CPU_INFO_FOREACH(cii, mci)) { 376 mspc = &mci->ci_schedstate; 377 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 378 kcpuset_isset(l->l_affinity, cpu_index(mci))) 379 break; 380 } 381 if (mci == NULL) { 382 lwp_unlock(l); 383 mutex_exit(proc_lock); 384 goto fail; 385 } 386 lwp_migrate(l, mci); 387 } 388 mutex_exit(proc_lock); 389 390 #ifdef __HAVE_MD_CPU_OFFLINE 391 cpu_offline_md(); 392 #endif 393 return; 394 fail: 395 /* Just unset the SPCF_OFFLINE flag, caller will check */ 396 s = splsched(); 397 spc->spc_flags &= ~SPCF_OFFLINE; 398 splx(s); 399 } 400 401 static void 402 cpu_xc_online(struct cpu_info *ci) 403 { 404 struct schedstate_percpu *spc; 405 int s; 406 407 spc = &ci->ci_schedstate; 408 s = splsched(); 409 spc->spc_flags &= ~SPCF_OFFLINE; 410 splx(s); 411 } 412 413 int 414 cpu_setstate(struct cpu_info *ci, bool online) 415 { 416 struct schedstate_percpu *spc; 417 CPU_INFO_ITERATOR cii; 418 struct cpu_info *ci2; 419 uint64_t where; 420 xcfunc_t func; 421 int nonline; 422 423 spc = &ci->ci_schedstate; 424 425 KASSERT(mutex_owned(&cpu_lock)); 426 427 if (online) { 428 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 429 return 0; 430 func = (xcfunc_t)cpu_xc_online; 431 ncpuonline++; 432 } else { 433 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 434 return 0; 435 nonline = 0; 436 /* 437 * Ensure that at least one CPU within the processor set 438 * stays online. Revisit this later. 439 */ 440 for (CPU_INFO_FOREACH(cii, ci2)) { 441 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 442 continue; 443 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 444 continue; 445 nonline++; 446 } 447 if (nonline == 1) 448 return EBUSY; 449 func = (xcfunc_t)cpu_xc_offline; 450 ncpuonline--; 451 } 452 453 where = xc_unicast(0, func, ci, NULL, ci); 454 xc_wait(where); 455 if (online) { 456 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 457 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 458 /* If was not set offline, then it is busy */ 459 return EBUSY; 460 } 461 462 spc->spc_lastmod = time_second; 463 return 0; 464 } 465 466 #ifdef __HAVE_INTR_CONTROL 467 static void 468 cpu_xc_intr(struct cpu_info *ci) 469 { 470 struct schedstate_percpu *spc; 471 int s; 472 473 spc = &ci->ci_schedstate; 474 s = splsched(); 475 spc->spc_flags &= ~SPCF_NOINTR; 476 splx(s); 477 } 478 479 static void 480 cpu_xc_nointr(struct cpu_info *ci) 481 { 482 struct schedstate_percpu *spc; 483 int s; 484 485 spc = &ci->ci_schedstate; 486 s = splsched(); 487 spc->spc_flags |= SPCF_NOINTR; 488 splx(s); 489 } 490 491 int 492 cpu_setintr(struct cpu_info *ci, bool intr) 493 { 494 struct schedstate_percpu *spc; 495 CPU_INFO_ITERATOR cii; 496 struct cpu_info *ci2; 497 uint64_t where; 498 xcfunc_t func; 499 int nintr; 500 501 spc = &ci->ci_schedstate; 502 503 KASSERT(mutex_owned(&cpu_lock)); 504 505 if (intr) { 506 if ((spc->spc_flags & SPCF_NOINTR) == 0) 507 return 0; 508 func = (xcfunc_t)cpu_xc_intr; 509 } else { 510 if ((spc->spc_flags & SPCF_NOINTR) != 0) 511 return 0; 512 /* 513 * Ensure that at least one CPU within the system 514 * is handing device interrupts. 515 */ 516 nintr = 0; 517 for (CPU_INFO_FOREACH(cii, ci2)) { 518 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 519 continue; 520 if (ci2 == ci) 521 continue; 522 nintr++; 523 } 524 if (nintr == 0) 525 return EBUSY; 526 func = (xcfunc_t)cpu_xc_nointr; 527 } 528 529 where = xc_unicast(0, func, ci, NULL, ci); 530 xc_wait(where); 531 if (intr) { 532 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 533 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 534 /* If was not set offline, then it is busy */ 535 return EBUSY; 536 } 537 538 /* Direct interrupts away from the CPU and record the change. */ 539 cpu_intr_redistribute(); 540 spc->spc_lastmod = time_second; 541 return 0; 542 } 543 #else /* __HAVE_INTR_CONTROL */ 544 int 545 cpu_setintr(struct cpu_info *ci, bool intr) 546 { 547 548 return EOPNOTSUPP; 549 } 550 551 u_int 552 cpu_intr_count(struct cpu_info *ci) 553 { 554 555 return 0; /* 0 == "don't know" */ 556 } 557 #endif /* __HAVE_INTR_CONTROL */ 558 559 bool 560 cpu_softintr_p(void) 561 { 562 563 return (curlwp->l_pflag & LP_INTR) != 0; 564 } 565 566 #ifdef CPU_UCODE 567 int 568 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 569 { 570 firmware_handle_t fwh; 571 int error; 572 573 if (sc->sc_blob != NULL) { 574 firmware_free(sc->sc_blob, 0); 575 sc->sc_blob = NULL; 576 sc->sc_blobsize = 0; 577 } 578 579 error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname); 580 if (error != 0) { 581 aprint_error("ucode: firmware_open failed: %i\n", error); 582 goto err0; 583 } 584 585 sc->sc_blobsize = firmware_get_size(fwh); 586 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 587 if (sc->sc_blob == NULL) { 588 error = ENOMEM; 589 firmware_close(fwh); 590 goto err0; 591 } 592 593 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 594 firmware_close(fwh); 595 if (error != 0) 596 goto err1; 597 598 return 0; 599 600 err1: 601 firmware_free(sc->sc_blob, 0); 602 sc->sc_blob = NULL; 603 sc->sc_blobsize = 0; 604 err0: 605 return error; 606 } 607 #endif 608
Indexes created Mon Sep 22 21:09:42 GMT 2025