kern_cpu.c revision 1.48
1 /* $NetBSD: kern_cpu.c,v 1.48 2011/08/07 13:33:01 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.48 2011/08/07 13:33:01 rmind Exp $"); 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/idle.h> 64 #include <sys/sched.h> 65 #include <sys/intr.h> 66 #include <sys/conf.h> 67 #include <sys/cpu.h> 68 #include <sys/cpuio.h> 69 #include <sys/proc.h> 70 #include <sys/percpu.h> 71 #include <sys/kernel.h> 72 #include <sys/kauth.h> 73 #include <sys/xcall.h> 74 #include <sys/pool.h> 75 #include <sys/kmem.h> 76 #include <sys/select.h> 77 #include <sys/namei.h> 78 #include <sys/callout.h> 79 80 #include <uvm/uvm_extern.h> 81 82 /* 83 * If the port has state that cpu_data is the first thing in cpu_info, 84 * verify the claim is true. This will prevent the from getting out 85 * of sync. 86 */ 87 #ifdef __HAVE_CPU_DATA_FIRST 88 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 89 #else 90 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 91 #endif 92 93 void cpuctlattach(int); 94 95 static void cpu_xc_online(struct cpu_info *); 96 static void cpu_xc_offline(struct cpu_info *); 97 98 dev_type_ioctl(cpuctl_ioctl); 99 100 const struct cdevsw cpuctl_cdevsw = { 101 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 102 nullstop, notty, nopoll, nommap, nokqfilter, 103 D_OTHER | D_MPSAFE 104 }; 105 106 kmutex_t cpu_lock __cacheline_aligned; 107 int ncpu __read_mostly; 108 int ncpuonline __read_mostly; 109 bool mp_online __read_mostly; 110 111 kcpuset_t * kcpuset_attached __read_mostly; 112 113 struct cpuqueue cpu_queue __cacheline_aligned 114 = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 115 116 static struct cpu_info **cpu_infos __read_mostly; 117 118 int 119 mi_cpu_attach(struct cpu_info *ci) 120 { 121 int error; 122 123 KASSERT(maxcpus > 0); 124 125 ci->ci_index = ncpu; 126 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 127 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 128 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 129 130 /* This is useful for eg, per-cpu evcnt */ 131 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 132 cpu_index(ci)); 133 134 if (__predict_false(cpu_infos == NULL)) { 135 cpu_infos = 136 kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP); 137 kcpuset_create(&kcpuset_attached); 138 kcpuset_zero(kcpuset_attached); 139 } 140 cpu_infos[cpu_index(ci)] = ci; 141 kcpuset_set(kcpuset_attached, ci->ci_index); 142 143 sched_cpuattach(ci); 144 145 error = create_idle_lwp(ci); 146 if (error != 0) { 147 /* XXX revert sched_cpuattach */ 148 return error; 149 } 150 151 if (ci == curcpu()) 152 ci->ci_data.cpu_onproc = curlwp; 153 else 154 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 155 156 percpu_init_cpu(ci); 157 softint_init(ci); 158 callout_init_cpu(ci); 159 xc_init_cpu(ci); 160 pool_cache_cpu_init(ci); 161 selsysinit(ci); 162 cache_cpu_init(ci); 163 TAILQ_INIT(&ci->ci_data.cpu_biodone); 164 ncpu++; 165 ncpuonline++; 166 167 return 0; 168 } 169 170 void 171 cpuctlattach(int dummy) 172 { 173 174 KASSERT(cpu_infos != NULL); 175 } 176 177 int 178 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 179 { 180 CPU_INFO_ITERATOR cii; 181 cpustate_t *cs; 182 struct cpu_info *ci; 183 int error, i; 184 u_int id; 185 186 error = 0; 187 188 mutex_enter(&cpu_lock); 189 switch (cmd) { 190 case IOC_CPU_SETSTATE: 191 if (error == 0) 192 cs = data; 193 error = kauth_authorize_system(l->l_cred, 194 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 195 NULL); 196 if (error != 0) 197 break; 198 if (cs->cs_id >= maxcpus || 199 (ci = cpu_lookup(cs->cs_id)) == NULL) { 200 error = ESRCH; 201 break; 202 } 203 error = cpu_setintr(ci, cs->cs_intr); 204 error = cpu_setstate(ci, cs->cs_online); 205 break; 206 207 case IOC_CPU_GETSTATE: 208 if (error == 0) 209 cs = data; 210 id = cs->cs_id; 211 memset(cs, 0, sizeof(*cs)); 212 cs->cs_id = id; 213 if (cs->cs_id >= maxcpus || 214 (ci = cpu_lookup(id)) == NULL) { 215 error = ESRCH; 216 break; 217 } 218 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 219 cs->cs_online = false; 220 else 221 cs->cs_online = true; 222 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 223 cs->cs_intr = false; 224 else 225 cs->cs_intr = true; 226 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 227 cs->cs_lastmodhi = (int32_t) 228 (ci->ci_schedstate.spc_lastmod >> 32); 229 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 230 break; 231 232 case IOC_CPU_MAPID: 233 i = 0; 234 for (CPU_INFO_FOREACH(cii, ci)) { 235 if (i++ == *(int *)data) 236 break; 237 } 238 if (ci == NULL) 239 error = ESRCH; 240 else 241 *(int *)data = cpu_index(ci); 242 break; 243 244 case IOC_CPU_GETCOUNT: 245 *(int *)data = ncpu; 246 break; 247 248 default: 249 error = ENOTTY; 250 break; 251 } 252 mutex_exit(&cpu_lock); 253 254 return error; 255 } 256 257 struct cpu_info * 258 cpu_lookup(u_int idx) 259 { 260 struct cpu_info *ci; 261 262 KASSERT(idx < maxcpus); 263 264 if (__predict_false(cpu_infos == NULL)) { 265 KASSERT(idx == 0); 266 return curcpu(); 267 } 268 269 ci = cpu_infos[idx]; 270 KASSERT(ci == NULL || cpu_index(ci) == idx); 271 272 return ci; 273 } 274 275 static void 276 cpu_xc_offline(struct cpu_info *ci) 277 { 278 struct schedstate_percpu *spc, *mspc = NULL; 279 struct cpu_info *target_ci; 280 struct lwp *l; 281 CPU_INFO_ITERATOR cii; 282 int s; 283 284 /* 285 * Thread that made the cross call (separate context) holds 286 * cpu_lock on our behalf. 287 */ 288 spc = &ci->ci_schedstate; 289 s = splsched(); 290 spc->spc_flags |= SPCF_OFFLINE; 291 splx(s); 292 293 /* Take the first available CPU for the migration. */ 294 for (CPU_INFO_FOREACH(cii, target_ci)) { 295 mspc = &target_ci->ci_schedstate; 296 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 297 break; 298 } 299 KASSERT(target_ci != NULL); 300 301 /* 302 * Migrate all non-bound threads to the other CPU. Note that this 303 * runs from the xcall thread, thus handling of LSONPROC is not needed. 304 */ 305 mutex_enter(proc_lock); 306 LIST_FOREACH(l, &alllwp, l_list) { 307 struct cpu_info *mci; 308 309 lwp_lock(l); 310 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 311 lwp_unlock(l); 312 continue; 313 } 314 /* Normal case - no affinity */ 315 if ((l->l_flag & LW_AFFINITY) == 0) { 316 lwp_migrate(l, target_ci); 317 continue; 318 } 319 /* Affinity is set, find an online CPU in the set */ 320 KASSERT(l->l_affinity != NULL); 321 for (CPU_INFO_FOREACH(cii, mci)) { 322 mspc = &mci->ci_schedstate; 323 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 324 kcpuset_isset(l->l_affinity, cpu_index(mci))) 325 break; 326 } 327 if (mci == NULL) { 328 lwp_unlock(l); 329 mutex_exit(proc_lock); 330 goto fail; 331 } 332 lwp_migrate(l, mci); 333 } 334 mutex_exit(proc_lock); 335 336 #ifdef __HAVE_MD_CPU_OFFLINE 337 cpu_offline_md(); 338 #endif 339 return; 340 fail: 341 /* Just unset the SPCF_OFFLINE flag, caller will check */ 342 s = splsched(); 343 spc->spc_flags &= ~SPCF_OFFLINE; 344 splx(s); 345 } 346 347 static void 348 cpu_xc_online(struct cpu_info *ci) 349 { 350 struct schedstate_percpu *spc; 351 int s; 352 353 spc = &ci->ci_schedstate; 354 s = splsched(); 355 spc->spc_flags &= ~SPCF_OFFLINE; 356 splx(s); 357 } 358 359 int 360 cpu_setstate(struct cpu_info *ci, bool online) 361 { 362 struct schedstate_percpu *spc; 363 CPU_INFO_ITERATOR cii; 364 struct cpu_info *ci2; 365 uint64_t where; 366 xcfunc_t func; 367 int nonline; 368 369 spc = &ci->ci_schedstate; 370 371 KASSERT(mutex_owned(&cpu_lock)); 372 373 if (online) { 374 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 375 return 0; 376 func = (xcfunc_t)cpu_xc_online; 377 ncpuonline++; 378 } else { 379 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 380 return 0; 381 nonline = 0; 382 /* 383 * Ensure that at least one CPU within the processor set 384 * stays online. Revisit this later. 385 */ 386 for (CPU_INFO_FOREACH(cii, ci2)) { 387 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 388 continue; 389 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 390 continue; 391 nonline++; 392 } 393 if (nonline == 1) 394 return EBUSY; 395 func = (xcfunc_t)cpu_xc_offline; 396 ncpuonline--; 397 } 398 399 where = xc_unicast(0, func, ci, NULL, ci); 400 xc_wait(where); 401 if (online) { 402 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 403 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 404 /* If was not set offline, then it is busy */ 405 return EBUSY; 406 } 407 408 spc->spc_lastmod = time_second; 409 return 0; 410 } 411 412 #ifdef __HAVE_INTR_CONTROL 413 static void 414 cpu_xc_intr(struct cpu_info *ci) 415 { 416 struct schedstate_percpu *spc; 417 int s; 418 419 spc = &ci->ci_schedstate; 420 s = splsched(); 421 spc->spc_flags &= ~SPCF_NOINTR; 422 splx(s); 423 } 424 425 static void 426 cpu_xc_nointr(struct cpu_info *ci) 427 { 428 struct schedstate_percpu *spc; 429 int s; 430 431 spc = &ci->ci_schedstate; 432 s = splsched(); 433 spc->spc_flags |= SPCF_NOINTR; 434 splx(s); 435 } 436 437 int 438 cpu_setintr(struct cpu_info *ci, bool intr) 439 { 440 struct schedstate_percpu *spc; 441 CPU_INFO_ITERATOR cii; 442 struct cpu_info *ci2; 443 uint64_t where; 444 xcfunc_t func; 445 int nintr; 446 447 spc = &ci->ci_schedstate; 448 449 KASSERT(mutex_owned(&cpu_lock)); 450 451 if (intr) { 452 if ((spc->spc_flags & SPCF_NOINTR) == 0) 453 return 0; 454 func = (xcfunc_t)cpu_xc_intr; 455 } else { 456 if ((spc->spc_flags & SPCF_NOINTR) != 0) 457 return 0; 458 /* 459 * Ensure that at least one CPU within the system 460 * is handing device interrupts. 461 */ 462 nintr = 0; 463 for (CPU_INFO_FOREACH(cii, ci2)) { 464 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 465 continue; 466 if (ci2 == ci) 467 continue; 468 nintr++; 469 } 470 if (nintr == 0) 471 return EBUSY; 472 func = (xcfunc_t)cpu_xc_nointr; 473 } 474 475 where = xc_unicast(0, func, ci, NULL, ci); 476 xc_wait(where); 477 if (intr) { 478 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 479 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 480 /* If was not set offline, then it is busy */ 481 return EBUSY; 482 } 483 484 /* Direct interrupts away from the CPU and record the change. */ 485 cpu_intr_redistribute(); 486 spc->spc_lastmod = time_second; 487 return 0; 488 } 489 #else /* __HAVE_INTR_CONTROL */ 490 int 491 cpu_setintr(struct cpu_info *ci, bool intr) 492 { 493 494 return EOPNOTSUPP; 495 } 496 497 u_int 498 cpu_intr_count(struct cpu_info *ci) 499 { 500 501 return 0; /* 0 == "don't know" */ 502 } 503 #endif /* __HAVE_INTR_CONTROL */ 504 505 bool 506 cpu_softintr_p(void) 507 { 508 509 return (curlwp->l_pflag & LP_INTR) != 0; 510 } 511
Indexes created Mon Sep 22 05:09:51 GMT 2025