kern_cpu.c revision 1.63
1 /* $NetBSD: kern_cpu.c,v 1.63 2014/03/16 05:20:30 dholland Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010, 2012 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.63 2014/03/16 05:20:30 dholland Exp $"); 60 61 #include "opt_cpu_ucode.h" 62 #include "opt_compat_netbsd.h" 63 64 #include <sys/param.h> 65 #include <sys/systm.h> 66 #include <sys/idle.h> 67 #include <sys/sched.h> 68 #include <sys/intr.h> 69 #include <sys/conf.h> 70 #include <sys/cpu.h> 71 #include <sys/cpuio.h> 72 #include <sys/proc.h> 73 #include <sys/percpu.h> 74 #include <sys/kernel.h> 75 #include <sys/kauth.h> 76 #include <sys/xcall.h> 77 #include <sys/pool.h> 78 #include <sys/kmem.h> 79 #include <sys/select.h> 80 #include <sys/namei.h> 81 #include <sys/callout.h> 82 #include <sys/pcu.h> 83 84 #include <uvm/uvm_extern.h> 85 86 /* 87 * If the port has stated that cpu_data is the first thing in cpu_info, 88 * verify that the claim is true. This will prevent them from getting out 89 * of sync. 90 */ 91 #ifdef __HAVE_CPU_DATA_FIRST 92 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 93 #else 94 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 95 #endif 96 97 void cpuctlattach(int); 98 99 static void cpu_xc_online(struct cpu_info *); 100 static void cpu_xc_offline(struct cpu_info *); 101 102 dev_type_ioctl(cpuctl_ioctl); 103 104 const struct cdevsw cpuctl_cdevsw = { 105 .d_open = nullopen, 106 .d_close = nullclose, 107 .d_read = nullread, 108 .d_write = nullwrite, 109 .d_ioctl = cpuctl_ioctl, 110 .d_stop = nullstop, 111 .d_tty = notty, 112 .d_poll = nopoll, 113 .d_mmap = nommap, 114 .d_kqfilter = nokqfilter, 115 .d_flag = D_OTHER | D_MPSAFE 116 }; 117 118 kmutex_t cpu_lock __cacheline_aligned; 119 int ncpu __read_mostly; 120 int ncpuonline __read_mostly; 121 bool mp_online __read_mostly; 122 123 /* An array of CPUs. There are ncpu entries. */ 124 struct cpu_info **cpu_infos __read_mostly; 125 126 /* Note: set on mi_cpu_attach() and idle_loop(). */ 127 kcpuset_t * kcpuset_attached __read_mostly = NULL; 128 kcpuset_t * kcpuset_running __read_mostly = NULL; 129 130 /* 131 * mi_cpu_init: early initialisation of MI CPU related structures. 132 * 133 * Note: may not block and memory allocator is not yet available. 134 */ 135 void 136 mi_cpu_init(void) 137 { 138 139 mutex_init(&cpu_lock, MUTEX_DEFAULT, IPL_NONE); 140 141 kcpuset_create(&kcpuset_attached, true); 142 kcpuset_create(&kcpuset_running, true); 143 kcpuset_set(kcpuset_running, 0); 144 } 145 146 int 147 mi_cpu_attach(struct cpu_info *ci) 148 { 149 int error; 150 151 KASSERT(maxcpus > 0); 152 153 ci->ci_index = ncpu; 154 kcpuset_set(kcpuset_attached, cpu_index(ci)); 155 156 /* 157 * Create a convenience cpuset of just ourselves. 158 */ 159 kcpuset_create(&ci->ci_data.cpu_kcpuset, true); 160 kcpuset_set(ci->ci_data.cpu_kcpuset, cpu_index(ci)); 161 162 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 163 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 164 165 /* This is useful for eg, per-cpu evcnt */ 166 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 167 cpu_index(ci)); 168 169 if (__predict_false(cpu_infos == NULL)) { 170 size_t ci_bufsize = (maxcpus + 1) * sizeof(struct cpu_info *); 171 cpu_infos = kmem_zalloc(ci_bufsize, KM_SLEEP); 172 } 173 cpu_infos[cpu_index(ci)] = ci; 174 175 sched_cpuattach(ci); 176 177 error = create_idle_lwp(ci); 178 if (error != 0) { 179 /* XXX revert sched_cpuattach */ 180 return error; 181 } 182 183 if (ci == curcpu()) 184 ci->ci_data.cpu_onproc = curlwp; 185 else 186 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 187 188 percpu_init_cpu(ci); 189 softint_init(ci); 190 callout_init_cpu(ci); 191 xc_init_cpu(ci); 192 pool_cache_cpu_init(ci); 193 selsysinit(ci); 194 cache_cpu_init(ci); 195 TAILQ_INIT(&ci->ci_data.cpu_biodone); 196 ncpu++; 197 ncpuonline++; 198 199 return 0; 200 } 201 202 void 203 cpuctlattach(int dummy) 204 { 205 206 KASSERT(cpu_infos != NULL); 207 } 208 209 int 210 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 211 { 212 CPU_INFO_ITERATOR cii; 213 cpustate_t *cs; 214 struct cpu_info *ci; 215 int error, i; 216 u_int id; 217 218 error = 0; 219 220 mutex_enter(&cpu_lock); 221 switch (cmd) { 222 case IOC_CPU_SETSTATE: 223 cs = data; 224 error = kauth_authorize_system(l->l_cred, 225 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 226 NULL); 227 if (error != 0) 228 break; 229 if (cs->cs_id >= maxcpus || 230 (ci = cpu_lookup(cs->cs_id)) == NULL) { 231 error = ESRCH; 232 break; 233 } 234 cpu_setintr(ci, cs->cs_intr); 235 error = cpu_setstate(ci, cs->cs_online); 236 break; 237 238 case IOC_CPU_GETSTATE: 239 cs = data; 240 id = cs->cs_id; 241 memset(cs, 0, sizeof(*cs)); 242 cs->cs_id = id; 243 if (cs->cs_id >= maxcpus || 244 (ci = cpu_lookup(id)) == NULL) { 245 error = ESRCH; 246 break; 247 } 248 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 249 cs->cs_online = false; 250 else 251 cs->cs_online = true; 252 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 253 cs->cs_intr = false; 254 else 255 cs->cs_intr = true; 256 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 257 cs->cs_lastmodhi = (int32_t) 258 (ci->ci_schedstate.spc_lastmod >> 32); 259 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 260 cs->cs_hwid = ci->ci_cpuid; 261 break; 262 263 case IOC_CPU_MAPID: 264 i = 0; 265 for (CPU_INFO_FOREACH(cii, ci)) { 266 if (i++ == *(int *)data) 267 break; 268 } 269 if (ci == NULL) 270 error = ESRCH; 271 else 272 *(int *)data = cpu_index(ci); 273 break; 274 275 case IOC_CPU_GETCOUNT: 276 *(int *)data = ncpu; 277 break; 278 279 #ifdef CPU_UCODE 280 case IOC_CPU_UCODE_GET_VERSION: 281 error = cpu_ucode_get_version((struct cpu_ucode_version *)data); 282 break; 283 284 #ifdef COMPAT_60 285 case OIOC_CPU_UCODE_GET_VERSION: 286 error = compat6_cpu_ucode_get_version((struct compat6_cpu_ucode *)data); 287 break; 288 #endif 289 290 case IOC_CPU_UCODE_APPLY: 291 error = kauth_authorize_machdep(l->l_cred, 292 KAUTH_MACHDEP_CPU_UCODE_APPLY, 293 NULL, NULL, NULL, NULL); 294 if (error != 0) 295 break; 296 error = cpu_ucode_apply((const struct cpu_ucode *)data); 297 break; 298 299 #ifdef COMPAT_60 300 case OIOC_CPU_UCODE_APPLY: 301 error = kauth_authorize_machdep(l->l_cred, 302 KAUTH_MACHDEP_CPU_UCODE_APPLY, 303 NULL, NULL, NULL, NULL); 304 if (error != 0) 305 break; 306 error = compat6_cpu_ucode_apply((const struct compat6_cpu_ucode *)data); 307 break; 308 #endif 309 #endif 310 311 default: 312 error = ENOTTY; 313 break; 314 } 315 mutex_exit(&cpu_lock); 316 317 return error; 318 } 319 320 struct cpu_info * 321 cpu_lookup(u_int idx) 322 { 323 struct cpu_info *ci; 324 325 KASSERT(idx < maxcpus); 326 327 if (__predict_false(cpu_infos == NULL)) { 328 KASSERT(idx == 0); 329 return curcpu(); 330 } 331 332 ci = cpu_infos[idx]; 333 KASSERT(ci == NULL || cpu_index(ci) == idx); 334 335 return ci; 336 } 337 338 static void 339 cpu_xc_offline(struct cpu_info *ci) 340 { 341 struct schedstate_percpu *spc, *mspc = NULL; 342 struct cpu_info *target_ci; 343 struct lwp *l; 344 CPU_INFO_ITERATOR cii; 345 int s; 346 347 /* 348 * Thread that made the cross call (separate context) holds 349 * cpu_lock on our behalf. 350 */ 351 spc = &ci->ci_schedstate; 352 s = splsched(); 353 spc->spc_flags |= SPCF_OFFLINE; 354 splx(s); 355 356 /* Take the first available CPU for the migration. */ 357 for (CPU_INFO_FOREACH(cii, target_ci)) { 358 mspc = &target_ci->ci_schedstate; 359 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 360 break; 361 } 362 KASSERT(target_ci != NULL); 363 364 /* 365 * Migrate all non-bound threads to the other CPU. Note that this 366 * runs from the xcall thread, thus handling of LSONPROC is not needed. 367 */ 368 mutex_enter(proc_lock); 369 LIST_FOREACH(l, &alllwp, l_list) { 370 struct cpu_info *mci; 371 372 lwp_lock(l); 373 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 374 lwp_unlock(l); 375 continue; 376 } 377 /* Regular case - no affinity. */ 378 if (l->l_affinity == NULL) { 379 lwp_migrate(l, target_ci); 380 continue; 381 } 382 /* Affinity is set, find an online CPU in the set. */ 383 for (CPU_INFO_FOREACH(cii, mci)) { 384 mspc = &mci->ci_schedstate; 385 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 386 kcpuset_isset(l->l_affinity, cpu_index(mci))) 387 break; 388 } 389 if (mci == NULL) { 390 lwp_unlock(l); 391 mutex_exit(proc_lock); 392 goto fail; 393 } 394 lwp_migrate(l, mci); 395 } 396 mutex_exit(proc_lock); 397 398 #if PCU_UNIT_COUNT > 0 399 pcu_save_all_on_cpu(); 400 #endif 401 402 #ifdef __HAVE_MD_CPU_OFFLINE 403 cpu_offline_md(); 404 #endif 405 return; 406 fail: 407 /* Just unset the SPCF_OFFLINE flag, caller will check */ 408 s = splsched(); 409 spc->spc_flags &= ~SPCF_OFFLINE; 410 splx(s); 411 } 412 413 static void 414 cpu_xc_online(struct cpu_info *ci) 415 { 416 struct schedstate_percpu *spc; 417 int s; 418 419 spc = &ci->ci_schedstate; 420 s = splsched(); 421 spc->spc_flags &= ~SPCF_OFFLINE; 422 splx(s); 423 } 424 425 int 426 cpu_setstate(struct cpu_info *ci, bool online) 427 { 428 struct schedstate_percpu *spc; 429 CPU_INFO_ITERATOR cii; 430 struct cpu_info *ci2; 431 uint64_t where; 432 xcfunc_t func; 433 int nonline; 434 435 spc = &ci->ci_schedstate; 436 437 KASSERT(mutex_owned(&cpu_lock)); 438 439 if (online) { 440 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 441 return 0; 442 func = (xcfunc_t)cpu_xc_online; 443 ncpuonline++; 444 } else { 445 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 446 return 0; 447 nonline = 0; 448 /* 449 * Ensure that at least one CPU within the processor set 450 * stays online. Revisit this later. 451 */ 452 for (CPU_INFO_FOREACH(cii, ci2)) { 453 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 454 continue; 455 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 456 continue; 457 nonline++; 458 } 459 if (nonline == 1) 460 return EBUSY; 461 func = (xcfunc_t)cpu_xc_offline; 462 ncpuonline--; 463 } 464 465 where = xc_unicast(0, func, ci, NULL, ci); 466 xc_wait(where); 467 if (online) { 468 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 469 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 470 /* If was not set offline, then it is busy */ 471 return EBUSY; 472 } 473 474 spc->spc_lastmod = time_second; 475 return 0; 476 } 477 478 #ifdef __HAVE_INTR_CONTROL 479 static void 480 cpu_xc_intr(struct cpu_info *ci) 481 { 482 struct schedstate_percpu *spc; 483 int s; 484 485 spc = &ci->ci_schedstate; 486 s = splsched(); 487 spc->spc_flags &= ~SPCF_NOINTR; 488 splx(s); 489 } 490 491 static void 492 cpu_xc_nointr(struct cpu_info *ci) 493 { 494 struct schedstate_percpu *spc; 495 int s; 496 497 spc = &ci->ci_schedstate; 498 s = splsched(); 499 spc->spc_flags |= SPCF_NOINTR; 500 splx(s); 501 } 502 503 int 504 cpu_setintr(struct cpu_info *ci, bool intr) 505 { 506 struct schedstate_percpu *spc; 507 CPU_INFO_ITERATOR cii; 508 struct cpu_info *ci2; 509 uint64_t where; 510 xcfunc_t func; 511 int nintr; 512 513 spc = &ci->ci_schedstate; 514 515 KASSERT(mutex_owned(&cpu_lock)); 516 517 if (intr) { 518 if ((spc->spc_flags & SPCF_NOINTR) == 0) 519 return 0; 520 func = (xcfunc_t)cpu_xc_intr; 521 } else { 522 if ((spc->spc_flags & SPCF_NOINTR) != 0) 523 return 0; 524 /* 525 * Ensure that at least one CPU within the system 526 * is handing device interrupts. 527 */ 528 nintr = 0; 529 for (CPU_INFO_FOREACH(cii, ci2)) { 530 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 531 continue; 532 if (ci2 == ci) 533 continue; 534 nintr++; 535 } 536 if (nintr == 0) 537 return EBUSY; 538 func = (xcfunc_t)cpu_xc_nointr; 539 } 540 541 where = xc_unicast(0, func, ci, NULL, ci); 542 xc_wait(where); 543 if (intr) { 544 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 545 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 546 /* If was not set offline, then it is busy */ 547 return EBUSY; 548 } 549 550 /* Direct interrupts away from the CPU and record the change. */ 551 cpu_intr_redistribute(); 552 spc->spc_lastmod = time_second; 553 return 0; 554 } 555 #else /* __HAVE_INTR_CONTROL */ 556 int 557 cpu_setintr(struct cpu_info *ci, bool intr) 558 { 559 560 return EOPNOTSUPP; 561 } 562 563 u_int 564 cpu_intr_count(struct cpu_info *ci) 565 { 566 567 return 0; /* 0 == "don't know" */ 568 } 569 #endif /* __HAVE_INTR_CONTROL */ 570 571 bool 572 cpu_softintr_p(void) 573 { 574 575 return (curlwp->l_pflag & LP_INTR) != 0; 576 } 577 578 #ifdef CPU_UCODE 579 int 580 cpu_ucode_load(struct cpu_ucode_softc *sc, const char *fwname) 581 { 582 firmware_handle_t fwh; 583 int error; 584 585 if (sc->sc_blob != NULL) { 586 firmware_free(sc->sc_blob, 0); 587 sc->sc_blob = NULL; 588 sc->sc_blobsize = 0; 589 } 590 591 error = cpu_ucode_md_open(&fwh, sc->loader_version, fwname); 592 if (error != 0) { 593 aprint_error("ucode: firmware_open failed: %i\n", error); 594 goto err0; 595 } 596 597 sc->sc_blobsize = firmware_get_size(fwh); 598 sc->sc_blob = firmware_malloc(sc->sc_blobsize); 599 if (sc->sc_blob == NULL) { 600 error = ENOMEM; 601 firmware_close(fwh); 602 goto err0; 603 } 604 605 error = firmware_read(fwh, 0, sc->sc_blob, sc->sc_blobsize); 606 firmware_close(fwh); 607 if (error != 0) 608 goto err1; 609 610 return 0; 611 612 err1: 613 firmware_free(sc->sc_blob, 0); 614 sc->sc_blob = NULL; 615 sc->sc_blobsize = 0; 616 err0: 617 return error; 618 } 619 #endif 620
Indexes created Sat Sep 27 14:09:57 GMT 2025