kern_cpu.c revision 1.45
1 /* $NetBSD: kern_cpu.c,v 1.45 2010/12/22 02:43:23 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009, 2010 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.45 2010/12/22 02:43:23 matt Exp $"); 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/idle.h> 64 #include <sys/sched.h> 65 #include <sys/intr.h> 66 #include <sys/conf.h> 67 #include <sys/cpu.h> 68 #include <sys/cpuio.h> 69 #include <sys/proc.h> 70 #include <sys/percpu.h> 71 #include <sys/kernel.h> 72 #include <sys/kauth.h> 73 #include <sys/xcall.h> 74 #include <sys/pool.h> 75 #include <sys/kmem.h> 76 #include <sys/select.h> 77 #include <sys/namei.h> 78 #include <sys/callout.h> 79 80 #include <uvm/uvm_extern.h> 81 82 /* 83 * If the port has state that cpu_data is the first thing in cpu_info, 84 * verify the claim is true. This will prevent the from getting out 85 * of sync. 86 */ 87 #ifdef __HAVE_CPU_DATA_FIRST 88 CTASSERT(offsetof(struct cpu_info, ci_data) == 0); 89 #else 90 CTASSERT(offsetof(struct cpu_info, ci_data) != 0); 91 #endif 92 93 void cpuctlattach(int); 94 95 static void cpu_xc_online(struct cpu_info *); 96 static void cpu_xc_offline(struct cpu_info *); 97 98 dev_type_ioctl(cpuctl_ioctl); 99 100 const struct cdevsw cpuctl_cdevsw = { 101 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 102 nullstop, notty, nopoll, nommap, nokqfilter, 103 D_OTHER | D_MPSAFE 104 }; 105 106 kmutex_t cpu_lock; 107 int ncpu; 108 int ncpuonline; 109 bool mp_online; 110 struct cpuqueue cpu_queue = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 111 112 static struct cpu_info **cpu_infos; 113 114 int 115 mi_cpu_attach(struct cpu_info *ci) 116 { 117 int error; 118 119 KASSERT(maxcpus > 0); 120 121 ci->ci_index = ncpu; 122 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 123 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 124 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 125 126 /* This is useful for eg, per-cpu evcnt */ 127 snprintf(ci->ci_data.cpu_name, sizeof(ci->ci_data.cpu_name), "cpu%d", 128 cpu_index(ci)); 129 130 sched_cpuattach(ci); 131 132 error = create_idle_lwp(ci); 133 if (error != 0) { 134 /* XXX revert sched_cpuattach */ 135 return error; 136 } 137 138 if (ci == curcpu()) 139 ci->ci_data.cpu_onproc = curlwp; 140 else 141 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 142 143 percpu_init_cpu(ci); 144 softint_init(ci); 145 callout_init_cpu(ci); 146 xc_init_cpu(ci); 147 pool_cache_cpu_init(ci); 148 selsysinit(ci); 149 cache_cpu_init(ci); 150 TAILQ_INIT(&ci->ci_data.cpu_biodone); 151 ncpu++; 152 ncpuonline++; 153 154 if (cpu_infos == NULL) { 155 cpu_infos = 156 kmem_zalloc(sizeof(cpu_infos[0]) * maxcpus, KM_SLEEP); 157 } 158 cpu_infos[cpu_index(ci)] = ci; 159 160 return 0; 161 } 162 163 void 164 cpuctlattach(int dummy) 165 { 166 167 KASSERT(cpu_infos != NULL); 168 } 169 170 int 171 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 172 { 173 CPU_INFO_ITERATOR cii; 174 cpustate_t *cs; 175 struct cpu_info *ci; 176 int error, i; 177 u_int id; 178 179 error = 0; 180 181 mutex_enter(&cpu_lock); 182 switch (cmd) { 183 case IOC_CPU_SETSTATE: 184 if (error == 0) 185 cs = data; 186 error = kauth_authorize_system(l->l_cred, 187 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 188 NULL); 189 if (error != 0) 190 break; 191 if (cs->cs_id >= maxcpus || 192 (ci = cpu_lookup(cs->cs_id)) == NULL) { 193 error = ESRCH; 194 break; 195 } 196 error = cpu_setintr(ci, cs->cs_intr); 197 error = cpu_setstate(ci, cs->cs_online); 198 break; 199 200 case IOC_CPU_GETSTATE: 201 if (error == 0) 202 cs = data; 203 id = cs->cs_id; 204 memset(cs, 0, sizeof(*cs)); 205 cs->cs_id = id; 206 if (cs->cs_id >= maxcpus || 207 (ci = cpu_lookup(id)) == NULL) { 208 error = ESRCH; 209 break; 210 } 211 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 212 cs->cs_online = false; 213 else 214 cs->cs_online = true; 215 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 216 cs->cs_intr = false; 217 else 218 cs->cs_intr = true; 219 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 220 cs->cs_lastmodhi = (int32_t) 221 (ci->ci_schedstate.spc_lastmod >> 32); 222 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 223 break; 224 225 case IOC_CPU_MAPID: 226 i = 0; 227 for (CPU_INFO_FOREACH(cii, ci)) { 228 if (i++ == *(int *)data) 229 break; 230 } 231 if (ci == NULL) 232 error = ESRCH; 233 else 234 *(int *)data = cpu_index(ci); 235 break; 236 237 case IOC_CPU_GETCOUNT: 238 *(int *)data = ncpu; 239 break; 240 241 default: 242 error = ENOTTY; 243 break; 244 } 245 mutex_exit(&cpu_lock); 246 247 return error; 248 } 249 250 struct cpu_info * 251 cpu_lookup(u_int idx) 252 { 253 struct cpu_info *ci; 254 255 KASSERT(idx < maxcpus); 256 257 if (__predict_false(cpu_infos == NULL)) { 258 KASSERT(idx == 0); 259 return curcpu(); 260 } 261 262 ci = cpu_infos[idx]; 263 KASSERT(ci == NULL || cpu_index(ci) == idx); 264 265 return ci; 266 } 267 268 static void 269 cpu_xc_offline(struct cpu_info *ci) 270 { 271 struct schedstate_percpu *spc, *mspc = NULL; 272 struct cpu_info *target_ci; 273 struct lwp *l; 274 CPU_INFO_ITERATOR cii; 275 int s; 276 277 /* 278 * Thread that made the cross call (separate context) holds 279 * cpu_lock on our behalf. 280 */ 281 spc = &ci->ci_schedstate; 282 s = splsched(); 283 spc->spc_flags |= SPCF_OFFLINE; 284 splx(s); 285 286 /* Take the first available CPU for the migration. */ 287 for (CPU_INFO_FOREACH(cii, target_ci)) { 288 mspc = &target_ci->ci_schedstate; 289 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 290 break; 291 } 292 KASSERT(target_ci != NULL); 293 294 /* 295 * Migrate all non-bound threads to the other CPU. Note that this 296 * runs from the xcall thread, thus handling of LSONPROC is not needed. 297 */ 298 mutex_enter(proc_lock); 299 LIST_FOREACH(l, &alllwp, l_list) { 300 struct cpu_info *mci; 301 302 lwp_lock(l); 303 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 304 lwp_unlock(l); 305 continue; 306 } 307 /* Normal case - no affinity */ 308 if ((l->l_flag & LW_AFFINITY) == 0) { 309 lwp_migrate(l, target_ci); 310 continue; 311 } 312 /* Affinity is set, find an online CPU in the set */ 313 KASSERT(l->l_affinity != NULL); 314 for (CPU_INFO_FOREACH(cii, mci)) { 315 mspc = &mci->ci_schedstate; 316 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 317 kcpuset_isset(cpu_index(mci), l->l_affinity)) 318 break; 319 } 320 if (mci == NULL) { 321 lwp_unlock(l); 322 mutex_exit(proc_lock); 323 goto fail; 324 } 325 lwp_migrate(l, mci); 326 } 327 mutex_exit(proc_lock); 328 329 #ifdef __HAVE_MD_CPU_OFFLINE 330 cpu_offline_md(); 331 #endif 332 return; 333 fail: 334 /* Just unset the SPCF_OFFLINE flag, caller will check */ 335 s = splsched(); 336 spc->spc_flags &= ~SPCF_OFFLINE; 337 splx(s); 338 } 339 340 static void 341 cpu_xc_online(struct cpu_info *ci) 342 { 343 struct schedstate_percpu *spc; 344 int s; 345 346 spc = &ci->ci_schedstate; 347 s = splsched(); 348 spc->spc_flags &= ~SPCF_OFFLINE; 349 splx(s); 350 } 351 352 int 353 cpu_setstate(struct cpu_info *ci, bool online) 354 { 355 struct schedstate_percpu *spc; 356 CPU_INFO_ITERATOR cii; 357 struct cpu_info *ci2; 358 uint64_t where; 359 xcfunc_t func; 360 int nonline; 361 362 spc = &ci->ci_schedstate; 363 364 KASSERT(mutex_owned(&cpu_lock)); 365 366 if (online) { 367 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 368 return 0; 369 func = (xcfunc_t)cpu_xc_online; 370 ncpuonline++; 371 } else { 372 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 373 return 0; 374 nonline = 0; 375 /* 376 * Ensure that at least one CPU within the processor set 377 * stays online. Revisit this later. 378 */ 379 for (CPU_INFO_FOREACH(cii, ci2)) { 380 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 381 continue; 382 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 383 continue; 384 nonline++; 385 } 386 if (nonline == 1) 387 return EBUSY; 388 func = (xcfunc_t)cpu_xc_offline; 389 ncpuonline--; 390 } 391 392 where = xc_unicast(0, func, ci, NULL, ci); 393 xc_wait(where); 394 if (online) { 395 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 396 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 397 /* If was not set offline, then it is busy */ 398 return EBUSY; 399 } 400 401 spc->spc_lastmod = time_second; 402 return 0; 403 } 404 405 #ifdef __HAVE_INTR_CONTROL 406 static void 407 cpu_xc_intr(struct cpu_info *ci) 408 { 409 struct schedstate_percpu *spc; 410 int s; 411 412 spc = &ci->ci_schedstate; 413 s = splsched(); 414 spc->spc_flags &= ~SPCF_NOINTR; 415 splx(s); 416 } 417 418 static void 419 cpu_xc_nointr(struct cpu_info *ci) 420 { 421 struct schedstate_percpu *spc; 422 int s; 423 424 spc = &ci->ci_schedstate; 425 s = splsched(); 426 spc->spc_flags |= SPCF_NOINTR; 427 splx(s); 428 } 429 430 int 431 cpu_setintr(struct cpu_info *ci, bool intr) 432 { 433 struct schedstate_percpu *spc; 434 CPU_INFO_ITERATOR cii; 435 struct cpu_info *ci2; 436 uint64_t where; 437 xcfunc_t func; 438 int nintr; 439 440 spc = &ci->ci_schedstate; 441 442 KASSERT(mutex_owned(&cpu_lock)); 443 444 if (intr) { 445 if ((spc->spc_flags & SPCF_NOINTR) == 0) 446 return 0; 447 func = (xcfunc_t)cpu_xc_intr; 448 } else { 449 if ((spc->spc_flags & SPCF_NOINTR) != 0) 450 return 0; 451 /* 452 * Ensure that at least one CPU within the system 453 * is handing device interrupts. 454 */ 455 nintr = 0; 456 for (CPU_INFO_FOREACH(cii, ci2)) { 457 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 458 continue; 459 if (ci2 == ci) 460 continue; 461 nintr++; 462 } 463 if (nintr == 0) 464 return EBUSY; 465 func = (xcfunc_t)cpu_xc_nointr; 466 } 467 468 where = xc_unicast(0, func, ci, NULL, ci); 469 xc_wait(where); 470 if (intr) { 471 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 472 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 473 /* If was not set offline, then it is busy */ 474 return EBUSY; 475 } 476 477 /* Direct interrupts away from the CPU and record the change. */ 478 cpu_intr_redistribute(); 479 spc->spc_lastmod = time_second; 480 return 0; 481 } 482 #else /* __HAVE_INTR_CONTROL */ 483 int 484 cpu_setintr(struct cpu_info *ci, bool intr) 485 { 486 487 return EOPNOTSUPP; 488 } 489 490 u_int 491 cpu_intr_count(struct cpu_info *ci) 492 { 493 494 return 0; /* 0 == "don't know" */ 495 } 496 #endif /* __HAVE_INTR_CONTROL */ 497 498 bool 499 cpu_softintr_p(void) 500 { 501 502 return (curlwp->l_pflag & LP_INTR) != 0; 503 } 504
Indexes created Sun Sep 21 20:09:37 GMT 2025