kern_cpu.c revision 1.65
1 /* $NetBSD: kern_cpu.c,v 1.65 2014/03/25 12:50:53 macallan Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.65 2014/03/25 12:50:53 macallan Exp $"); 60 61 #include "opt_cpu_ucode.h" 62 #include "opt_compat_netbsd.h" 63 64 #include <sys/param.h> 65 #include <sys/systm.h> 66 #include <sys/idle.h> 67 #include <sys/sched.h> 68 #include <sys/intr.h> 69 #include <sys/conf.h> 70 #include <sys/cpu.h> 71 #include <sys/cpuio.h> 72 #include <sys/proc.h> 73 #include <sys/percpu.h> 74 #include <sys/kernel.h> 75 #include <sys/kauth.h> 76 #include <sys/xcall.h> 77 #include <sys/pool.h> 78 #include <sys/kmem.h> 79 #include <sys/select.h> 80 #include <sys/namei.h> 81 #include <sys/callout.h> 82 #include <sys/pcu.h> 83 84 #include <uvm/uvm_extern.h> 85 86 /* 87 * If the port has stated that cpu_data is the first thing in cpu_info, 88 * verify that the claim is true. This will prevent them from getting out 89 * of sync. 90 */ 91 #ifdef __HAVE_CPU_DATA_FIRST 92 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 93 #else 94 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 95 #endif 96 97 void cpuctlattach(int); 98 99 static void cpu_xc_online(struct cpu_info *); 100 static void cpu_xc_offline(struct cpu_info *); 101 102 dev_type_ioctl(cpuctl_ioctl); 103 104 const struct cdevsw cpuctl_cdevsw = { 105 .d_open = nullopen, 106 .d_close = nullclose, 107 .d_read = nullread, 108 .d_write = nullwrite, 109 .d_ioctl = cpuctl_ioctl, 110 .d_stop = nullstop, 111 .d_tty = notty, 112 .d_poll = nopoll, 113 .d_mmap = nommap, 114 .d_kqfilter = nokqfilter, 115 .d_flag = D_OTHER | D_MPSAFE 116 }; 117 118 kmutex_t cpu_lock __cacheline_aligned; 119 int ncpu __read_mostly; 120 int ncpuonline __read_mostly; 121 bool mp_online __read_mostly; 122 123 /* An array of CPUs. There are ncpu entries. */ 124 struct cpu_info **cpu_infos __read_mostly; 125 126 /* Note: set on mi_cpu_attach() and idle_loop(). */ 127 kcpuset_t * kcpuset_attached __read_mostly = NULL; 128 kcpuset_t * kcpuset_running __read_mostly = NULL; 129 130 131 static char cpu_model[128]; 132 133 /* 134 * mi_cpu_init: early initialisation of MI CPU related structures. 135 * 136 * Note: may not block and memory allocator is not yet available. 137 */ 138 void 139 mi_cpu_init(void) 140 { 141 142 mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE); 143 144 kcpuset_create(&kcpuset_attached, true); 145 kcpuset_create(&kcpuset_running, true); 146 kcpuset_set(kcpuset_running, 0); 147 } 148 149 int 150 mi_cpu_attach(struct cpu_info *ci) 151 { 152 int error; 153 154 KASSERT(maxcpus > 0); 155 156 ci->ci_index = ncpu; 157 kcpuset_set(kcpuset_attached, cpu_index(ci)); 158 159 /* 160 * Create a convenience cpuset of just ourselves. 161 */ 162 kcpuset_create(&ci->ci_data.cpu_kcpuset, true); 163 kcpuset_set(ci->ci_data.cpu_kcpuset, cpu_index(ci)); 164 165 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 166 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 167 168 /* This is useful for eg, per-cpu evcnt */ 169 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 170 cpu_index(ci)); 171 172 if (__predict_false(cpu_infos == NULL)) { 173 size_t ci_bufsize = (maxcpus + 1) * sizeof(struct cpu_info *); 174 cpu_infos = kmem_zalloc(ci_bufsize, KM_SLEEP); 175 } 176 cpu_infos[cpu_index(ci)] = ci; 177 178 sched_cpuattach(ci); 179 180 error = create_idle_lwp(ci); 181 if (error != 0) { 182 /* XXX revert sched_cpuattach */ 183 return error; 184 } 185 186 if (ci == curcpu()) 187 ci->ci_data.cpu_onproc = curlwp; 188 else 189 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 190 191 percpu_init_cpu(ci); 192 softint_init(ci); 193 callout_init_cpu(ci); 194 xc_init_cpu(ci); 195 pool_cache_cpu_init(ci); 196 selsysinit(ci); 197 cache_cpu_init(ci); 198 TAILQ_INIT(&ci->ci_data.cpu_biodone); 199 ncpu++; 200 ncpuonline++; 201 202 return 0; 203 } 204 205 void 206 cpuctlattach(int dummy) 207 { 208 209 KASSERT(cpu_infos != NULL); 210 } 211 212 int 213 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 214 { 215 CPU_INFO_ITERATOR cii; 216 cpustate_t *cs; 217 struct cpu_info *ci; 218 int error, i; 219 u_int id; 220 221 error = 0; 222 223 mutex_enter(&cpu_lock); 224 switch (cmd) { 225 case IOC_CPU_SETSTATE: 226 cs = data; 227 error = kauth_authorize_system(l->l_cred, 228 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 229 NULL); 230 if (error != 0) 231 break; 232 if (cs->cs_id >= maxcpus || 233 (ci = cpu_lookup(cs->cs_id)) == NULL) { 234 error = ESRCH; 235 break; 236 } 237 cpu_setintr(ci, cs->cs_intr); 238 error = cpu_setstate(ci, cs->cs_online); 239 break; 240 241 case IOC_CPU_GETSTATE: 242 cs = data; 243 id = cs->cs_id; 244 memset(cs, 0, sizeof(*cs)); 245 cs->cs_id = id; 246 if (cs->cs_id >= maxcpus || 247 (ci = cpu_lookup(id)) == NULL) { 248 error = ESRCH; 249 break; 250 } 251 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 252 cs->cs_online = false; 253 else 254 cs->cs_online = true; 255 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 256 cs->cs_intr = false; 257 else 258 cs->cs_intr = true; 259 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 260 cs->cs_lastmodhi = (int32_t) 261 (ci->ci_schedstate.spc_lastmod >> 32); 262 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 263 cs->cs_hwid = ci->ci_cpuid; 264 break; 265 266 case IOC_CPU_MAPID: 267 i = 0; 268 for (CPU_INFO_FOREACH(cii, ci)) { 269 if (i++ == *(int *)data) 270 break; 271 } 272 if (ci == NULL) 273 error = ESRCH; 274 else 275 *(int *)data = cpu_index(ci); 276 break; 277 278 case IOC_CPU_GETCOUNT: 279 *(int *)data = ncpu; 280 break; 281 282 #ifdef CPU_UCODE 283 case IOC_CPU_UCODE_GET_VERSION: 284 error = cpu_ucode_get_version((struct cpu_ucode_version *)data); 285 break; 286 287 #ifdef COMPAT_60 288 case OIOC_CPU_UCODE_GET_VERSION: 289 error = compat6_cpu_ucode_get_version((struct compat6_cpu_ucode *)data); 290 break; 291 #endif 292 293 case IOC_CPU_UCODE_APPLY: 294 error = kauth_authorize_machdep(l->l_cred, 295 KAUTH_MACHDEP_CPU_UCODE_APPLY, 296 NULL, NULL, NULL, NULL); 297 if (error != 0) 298 break; 299 error = cpu_ucode_apply((const struct cpu_ucode *)data); 300 break; 301 302 #ifdef COMPAT_60 303 case OIOC_CPU_UCODE_APPLY: 304 error = kauth_authorize_machdep(l->l_cred, 305 KAUTH_MACHDEP_CPU_UCODE_APPLY, 306 NULL, NULL, NULL, NULL); 307 if (error != 0) 308 break; 309 error = compat6_cpu_ucode_apply((const struct compat6_cpu_ucode *)data); 310 break; 311 #endif 312 #endif 313 314 default: 315 error = ENOTTY; 316 break; 317 } 318 mutex_exit(&cpu_lock); 319 320 return error; 321 } 322 323 struct cpu_info * 324 cpu_lookup(u_int idx) 325 { 326 struct cpu_info *ci; 327 328 KASSERT(idx < maxcpus); 329 330 if (__predict_false(cpu_infos == NULL)) { 331 KASSERT(idx == 0); 332 return curcpu(); 333 } 334 335 ci = cpu_infos[idx]; 336 KASSERT(ci == NULL || cpu_index(ci) == idx); 337 338 return ci; 339 } 340 341 static void 342 cpu_xc_offline(struct cpu_info *ci) 343 { 344 struct schedstate_percpu *spc, *mspc = NULL; 345 struct cpu_info *target_ci; 346 struct lwp *l; 347 CPU_INFO_ITERATOR cii; 348 int s; 349 350 /* 351 * Thread that made the cross call (separate context) holds 352 * cpu_lock on our behalf. 353 */ 354 spc = &ci->ci_schedstate; 355 s = splsched(); 356 spc->spc_flags |= SPCF_OFFLINE; 357 splx(s); 358 359 /* Take the first available CPU for the migration. */ 360 for (CPU_INFO_FOREACH(cii, target_ci)) { 361 mspc = &target_ci->ci_schedstate; 362 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 363 break; 364 } 365 KASSERT(target_ci != NULL); 366 367 /* 368 * Migrate all non-bound threads to the other CPU. Note that this 369 * runs from the xcall thread, thus handling of LSONPROC is not needed. 370 */ 371 mutex_enter(proc_lock); 372 LIST_FOREACH(l, &alllwp, l_list) { 373 struct cpu_info *mci; 374 375 lwp_lock(l); 376 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 377 lwp_unlock(l); 378 continue; 379 } 380 /* Regular case - no affinity. */ 381 if (l->l_affinity == NULL) { 382 lwp_migrate(l, target_ci); 383 continue; 384 } 385 /* Affinity is set, find an online CPU in the set. */ 386 for (CPU_INFO_FOREACH(cii, mci)) { 387 mspc = &mci->ci_schedstate; 388 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 389 kcpuset_isset(l->l_affinity, cpu_index(mci))) 390 break; 391 } 392 if (mci == NULL) { 393 lwp_unlock(l); 394 mutex_exit(proc_lock); 395 goto fail; 396 } 397 lwp_migrate(l, mci); 398 } 399 mutex_exit(proc_lock); 400 401 #if PCU_UNIT_COUNT > 0 402 pcu_save_all_on_cpu(); 403 #endif 404 405 #ifdef __HAVE_MD_CPU_OFFLINE 406 cpu_offline_md(); 407 #endif 408 return; 409 fail: 410 /* Just unset the SPCF_OFFLINE flag, caller will check */ 411 s = splsched(); 412 spc->spc_flags &= ~SPCF_OFFLINE; 413 splx(s); 414 } 415 416 static void 417 cpu_xc_online(struct cpu_info *ci) 418 { 419 struct schedstate_percpu *spc; 420 int s; 421 422 spc = &ci->ci_schedstate; 423 s = splsched(); 424 spc->spc_flags &= ~SPCF_OFFLINE; 425 splx(s); 426 } 427 428 int 429 cpu_setstate(struct cpu_info *ci, bool online) 430 { 431 struct schedstate_percpu *spc; 432 CPU_INFO_ITERATOR cii; 433 struct cpu_info *ci2; 434 uint64_t where; 435 xcfunc_t func; 436 int nonline; 437 438 spc = &ci->ci_schedstate; 439 440 KASSERT(mutex_owned(&cpu_lock)); 441 442 if (online) { 443 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 444 return 0; 445 func = (xcfunc_t)cpu_xc_online; 446 ncpuonline++; 447 } else { 448 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 449 return 0; 450 nonline = 0; 451 /* 452 * Ensure that at least one CPU within the processor set 453 * stays online. Revisit this later. 454 */ 455 for (CPU_INFO_FOREACH(cii, ci2)) { 456 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 457 continue; 458 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 459 continue; 460 nonline++; 461 } 462 if (nonline == 1) 463 return EBUSY; 464 func = (xcfunc_t)cpu_xc_offline; 465 ncpuonline--; 466 } 467 468 where = xc_unicast(0, func, ci, NULL, ci); 469 xc_wait(where); 470 if (online) { 471 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 472 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 473 /* If was not set offline, then it is busy */ 474 return EBUSY; 475 } 476 477 spc->spc_lastmod = time_second; 478 return 0; 479 } 480 481 int 482 cpu_setmodel(const char *fmt, ...) 483 { 484 int len; 485 va_list ap; 486 487 va_start(ap, fmt); 488 len = vsnprintf(cpu_model, sizeof(cpu_model), fmt, ap); 489 va_end(ap); 490 return len; 491 } 492 493 const char * 494 cpu_getmodel(void) 495 { 496 return cpu_model; 497 } 498 499 #ifdef __HAVE_INTR_CONTROL 500 static void 501 cpu_xc_intr(struct cpu_info *ci) 502 { 503 struct schedstate_percpu *spc; 504 int s; 505 506 spc = &ci->ci_schedstate; 507 s = splsched(); 508 spc->spc_flags &= ~SPCF_NOINTR; 509 splx(s); 510 } 511 512 static void 513 cpu_xc_nointr(struct cpu_info *ci) 514 { 515 struct schedstate_percpu *spc; 516 int s; 517 518 spc = &ci->ci_schedstate; 519 s = splsched(); 520 spc->spc_flags |= SPCF_NOINTR; 521 splx(s); 522 } 523 524 int 525 cpu_setintr(struct cpu_info *ci, bool intr) 526 { 527 struct schedstate_percpu *spc; 528 CPU_INFO_ITERATOR cii; 529 struct cpu_info *ci2; 530 uint64_t where; 531 xcfunc_t func; 532 int nintr; 533 534 spc = &ci->ci_schedstate; 535 536 KASSERT(mutex_owned(&cpu_lock)); 537 538 if (intr) { 539 if ((spc->spc_flags & SPCF_NOINTR) == 0) 540 return 0; 541 func = (xcfunc_t)cpu_xc_intr; 542 } else { 543 if ((spc->spc_flags & SPCF_NOINTR) != 0) 544 return 0; 545 /* 546 * Ensure that at least one CPU within the system 547 * is handing device interrupts. 548 */ 549 nintr = 0; 550 for (CPU_INFO_FOREACH(cii, ci2)) { 551 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 552 continue; 553 if (ci2 == ci) 554 continue; 555 nintr++; 556 } 557 if (nintr == 0) 558 return EBUSY; 559 func = (xcfunc_t)cpu_xc_nointr; 560 } 561 562 where = xc_unicast(0, func, ci, NULL, ci); 563 xc_wait(where); 564 if (intr) { 565 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 566 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 567 /* If was not set offline, then it is busy */ 568 return EBUSY; 569 } 570 571 /* Direct interrupts away from the CPU and record the change. */ 572 cpu_intr_redistribute(); 573 spc->spc_lastmod = time_second; 574 return 0; 575 } 576 #else /* __HAVE_INTR_CONTROL */ 577 int 578 cpu_setintr(struct cpu_info *ci, bool intr) 579 { 580 581 return EOPNOTSUPP; 582 } 583 584 u_int 585 cpu_intr_count(struct cpu_info *ci) 586 { 587 588 return 0; /* 0 == "don't know" */ 589 } 590 #endif /* __HAVE_INTR_CONTROL */ 591 592 bool 593 cpu_softintr_p(void) 594 { 595 596 return (curlwp->l_pflag & LP_INTR) != 0; 597 } 598 599 #ifdef CPU_UCODE 600 int 601 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 602 { 603 firmware_handle_t fwh; 604 int error; 605 606 if (sc->sc_blob != NULL) { 607 firmware_free(sc->sc_blob, 0); 608 sc->sc_blob = NULL; 609 sc->sc_blobsize = 0; 610 } 611 612 error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname); 613 if (error != 0) { 614 aprint_error("ucode: firmware_open failed: %i\n", error); 615 goto err0; 616 } 617 618 sc->sc_blobsize = firmware_get_size(fwh); 619 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 620 if (sc->sc_blob == NULL) { 621 error = ENOMEM; 622 firmware_close(fwh); 623 goto err0; 624 } 625 626 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 627 firmware_close(fwh); 628 if (error != 0) 629 goto err1; 630 631 return 0; 632 633 err1: 634 firmware_free(sc->sc_blob, 0); 635 sc->sc_blob = NULL; 636 sc->sc_blobsize = 0; 637 err0: 638 return error; 639 } 640 #endif 641
Indexes created Tue Sep 23 14:10:03 GMT 2025