kern_cpu.c revision 1.28.2.2
1 /* $NetBSD: kern_cpu.c,v 1.28.2.2 2009/05/04 08:13:46 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /*- 33 * Copyright (c)2007 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: kern_cpu.c,v 1.28.2.2 2009/05/04 08:13:46 yamt Exp $"); 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/idle.h> 64 #include <sys/sched.h> 65 #include <sys/intr.h> 66 #include <sys/conf.h> 67 #include <sys/cpu.h> 68 #include <sys/cpuio.h> 69 #include <sys/proc.h> 70 #include <sys/percpu.h> 71 #include <sys/kernel.h> 72 #include <sys/kauth.h> 73 #include <sys/xcall.h> 74 #include <sys/pool.h> 75 #include <sys/kmem.h> 76 #include <sys/select.h> 77 #include <sys/namei.h> 78 #include <sys/callout.h> 79 80 #include <uvm/uvm_extern.h> 81 82 void cpuctlattach(int); 83 84 static void cpu_xc_online(struct cpu_info *); 85 static void cpu_xc_offline(struct cpu_info *); 86 87 dev_type_ioctl(cpuctl_ioctl); 88 89 const struct cdevsw cpuctl_cdevsw = { 90 nullopen, nullclose, nullread, nullwrite, cpuctl_ioctl, 91 nullstop, notty, nopoll, nommap, nokqfilter, 92 D_OTHER | D_MPSAFE 93 }; 94 95 kmutex_t cpu_lock; 96 int ncpu; 97 int ncpuonline; 98 bool mp_online; 99 struct cpuqueue cpu_queue = CIRCLEQ_HEAD_INITIALIZER(cpu_queue); 100 101 static struct cpu_info *cpu_infos[MAXCPUS]; 102 103 int 104 mi_cpu_attach(struct cpu_info *ci) 105 { 106 int error; 107 108 ci->ci_index = ncpu; 109 cpu_infos[cpu_index(ci)] = ci; 110 CIRCLEQ_INSERT_TAIL(&cpu_queue, ci, ci_data.cpu_qchain); 111 TAILQ_INIT(&ci->ci_data.cpu_ld_locks); 112 __cpu_simple_lock_init(&ci->ci_data.cpu_ld_lock); 113 114 sched_cpuattach(ci); 115 116 error = create_idle_lwp(ci); 117 if (error != 0) { 118 /* XXX revert sched_cpuattach */ 119 return error; 120 } 121 122 if (ci == curcpu()) 123 ci->ci_data.cpu_onproc = curlwp; 124 else 125 ci->ci_data.cpu_onproc = ci->ci_data.cpu_idlelwp; 126 127 percpu_init_cpu(ci); 128 softint_init(ci); 129 callout_init_cpu(ci); 130 xc_init_cpu(ci); 131 pool_cache_cpu_init(ci); 132 selsysinit(ci); 133 cache_cpu_init(ci); 134 TAILQ_INIT(&ci->ci_data.cpu_biodone); 135 ncpu++; 136 ncpuonline++; 137 138 return 0; 139 } 140 141 void 142 cpuctlattach(int dummy) 143 { 144 145 } 146 147 int 148 cpuctl_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l) 149 { 150 CPU_INFO_ITERATOR cii; 151 cpustate_t *cs; 152 struct cpu_info *ci; 153 int error, i; 154 u_int id; 155 156 error = 0; 157 158 mutex_enter(&cpu_lock); 159 switch (cmd) { 160 case IOC_CPU_SETSTATE: 161 if (error == 0) 162 cs = data; 163 error = kauth_authorize_system(l->l_cred, 164 KAUTH_SYSTEM_CPU, KAUTH_REQ_SYSTEM_CPU_SETSTATE, cs, NULL, 165 NULL); 166 if (error != 0) 167 break; 168 if (cs->cs_id >= __arraycount(cpu_infos) || 169 (ci = cpu_lookup(cs->cs_id)) == NULL) { 170 error = ESRCH; 171 break; 172 } 173 error = cpu_setintr(ci, cs->cs_intr); 174 error = cpu_setstate(ci, cs->cs_online); 175 break; 176 177 case IOC_CPU_GETSTATE: 178 if (error == 0) 179 cs = data; 180 id = cs->cs_id; 181 memset(cs, 0, sizeof(*cs)); 182 cs->cs_id = id; 183 if (cs->cs_id >= __arraycount(cpu_infos) || 184 (ci = cpu_lookup(id)) == NULL) { 185 error = ESRCH; 186 break; 187 } 188 if ((ci->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 189 cs->cs_online = false; 190 else 191 cs->cs_online = true; 192 if ((ci->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 193 cs->cs_intr = false; 194 else 195 cs->cs_intr = true; 196 cs->cs_lastmod = (int32_t)ci->ci_schedstate.spc_lastmod; 197 cs->cs_lastmodhi = (int32_t) 198 (ci->ci_schedstate.spc_lastmod >> 32); 199 cs->cs_intrcnt = cpu_intr_count(ci) + 1; 200 break; 201 202 case IOC_CPU_MAPID: 203 i = 0; 204 for (CPU_INFO_FOREACH(cii, ci)) { 205 if (i++ == *(int *)data) 206 break; 207 } 208 if (ci == NULL) 209 error = ESRCH; 210 else 211 *(int *)data = cpu_index(ci); 212 break; 213 214 case IOC_CPU_GETCOUNT: 215 *(int *)data = ncpu; 216 break; 217 218 default: 219 error = ENOTTY; 220 break; 221 } 222 mutex_exit(&cpu_lock); 223 224 return error; 225 } 226 227 struct cpu_info * 228 cpu_lookup(u_int idx) 229 { 230 struct cpu_info *ci = cpu_infos[idx]; 231 232 KASSERT(idx < __arraycount(cpu_infos)); 233 KASSERT(ci == NULL || cpu_index(ci) == idx); 234 235 return ci; 236 } 237 238 static void 239 cpu_xc_offline(struct cpu_info *ci) 240 { 241 struct schedstate_percpu *spc, *mspc = NULL; 242 struct cpu_info *target_ci; 243 struct lwp *l; 244 CPU_INFO_ITERATOR cii; 245 int s; 246 247 /* 248 * Thread that made the cross call (separate context) holds 249 * cpu_lock on our behalf. 250 */ 251 spc = &ci->ci_schedstate; 252 s = splsched(); 253 spc->spc_flags |= SPCF_OFFLINE; 254 splx(s); 255 256 /* Take the first available CPU for the migration. */ 257 for (CPU_INFO_FOREACH(cii, target_ci)) { 258 mspc = &target_ci->ci_schedstate; 259 if ((mspc->spc_flags & SPCF_OFFLINE) == 0) 260 break; 261 } 262 KASSERT(target_ci != NULL); 263 264 /* 265 * Migrate all non-bound threads to the other CPU. Note that this 266 * runs from the xcall thread, thus handling of LSONPROC is not needed. 267 */ 268 mutex_enter(proc_lock); 269 LIST_FOREACH(l, &alllwp, l_list) { 270 struct cpu_info *mci; 271 272 lwp_lock(l); 273 if (l->l_cpu != ci || (l->l_pflag & (LP_BOUND | LP_INTR))) { 274 lwp_unlock(l); 275 continue; 276 } 277 /* Normal case - no affinity */ 278 if ((l->l_flag & LW_AFFINITY) == 0) { 279 lwp_migrate(l, target_ci); 280 continue; 281 } 282 /* Affinity is set, find an online CPU in the set */ 283 KASSERT(l->l_affinity != NULL); 284 for (CPU_INFO_FOREACH(cii, mci)) { 285 mspc = &mci->ci_schedstate; 286 if ((mspc->spc_flags & SPCF_OFFLINE) == 0 && 287 kcpuset_isset(cpu_index(mci), l->l_affinity)) 288 break; 289 } 290 if (mci == NULL) { 291 lwp_unlock(l); 292 mutex_exit(proc_lock); 293 goto fail; 294 } 295 lwp_migrate(l, mci); 296 } 297 mutex_exit(proc_lock); 298 299 #ifdef __HAVE_MD_CPU_OFFLINE 300 cpu_offline_md(); 301 #endif 302 return; 303 fail: 304 /* Just unset the SPCF_OFFLINE flag, caller will check */ 305 s = splsched(); 306 spc->spc_flags &= ~SPCF_OFFLINE; 307 splx(s); 308 } 309 310 static void 311 cpu_xc_online(struct cpu_info *ci) 312 { 313 struct schedstate_percpu *spc; 314 int s; 315 316 spc = &ci->ci_schedstate; 317 s = splsched(); 318 spc->spc_flags &= ~SPCF_OFFLINE; 319 splx(s); 320 } 321 322 int 323 cpu_setstate(struct cpu_info *ci, bool online) 324 { 325 struct schedstate_percpu *spc; 326 CPU_INFO_ITERATOR cii; 327 struct cpu_info *ci2; 328 uint64_t where; 329 xcfunc_t func; 330 int nonline; 331 332 spc = &ci->ci_schedstate; 333 334 KASSERT(mutex_owned(&cpu_lock)); 335 336 if (online) { 337 if ((spc->spc_flags & SPCF_OFFLINE) == 0) 338 return 0; 339 func = (xcfunc_t)cpu_xc_online; 340 ncpuonline++; 341 } else { 342 if ((spc->spc_flags & SPCF_OFFLINE) != 0) 343 return 0; 344 nonline = 0; 345 /* 346 * Ensure that at least one CPU within the processor set 347 * stays online. Revisit this later. 348 */ 349 for (CPU_INFO_FOREACH(cii, ci2)) { 350 if ((ci2->ci_schedstate.spc_flags & SPCF_OFFLINE) != 0) 351 continue; 352 if (ci2->ci_schedstate.spc_psid != spc->spc_psid) 353 continue; 354 nonline++; 355 } 356 if (nonline == 1) 357 return EBUSY; 358 func = (xcfunc_t)cpu_xc_offline; 359 ncpuonline--; 360 } 361 362 where = xc_unicast(0, func, ci, NULL, ci); 363 xc_wait(where); 364 if (online) { 365 KASSERT((spc->spc_flags & SPCF_OFFLINE) == 0); 366 } else if ((spc->spc_flags & SPCF_OFFLINE) == 0) { 367 /* If was not set offline, then it is busy */ 368 return EBUSY; 369 } 370 371 spc->spc_lastmod = time_second; 372 return 0; 373 } 374 375 #ifdef __HAVE_INTR_CONTROL 376 static void 377 cpu_xc_intr(struct cpu_info *ci) 378 { 379 struct schedstate_percpu *spc; 380 int s; 381 382 spc = &ci->ci_schedstate; 383 s = splsched(); 384 spc->spc_flags &= ~SPCF_NOINTR; 385 splx(s); 386 } 387 388 static void 389 cpu_xc_nointr(struct cpu_info *ci) 390 { 391 struct schedstate_percpu *spc; 392 int s; 393 394 spc = &ci->ci_schedstate; 395 s = splsched(); 396 spc->spc_flags |= SPCF_NOINTR; 397 splx(s); 398 } 399 400 int 401 cpu_setintr(struct cpu_info *ci, bool intr) 402 { 403 struct schedstate_percpu *spc; 404 CPU_INFO_ITERATOR cii; 405 struct cpu_info *ci2; 406 uint64_t where; 407 xcfunc_t func; 408 int nintr; 409 410 spc = &ci->ci_schedstate; 411 412 KASSERT(mutex_owned(&cpu_lock)); 413 414 if (intr) { 415 if ((spc->spc_flags & SPCF_NOINTR) == 0) 416 return 0; 417 func = (xcfunc_t)cpu_xc_intr; 418 } else { 419 if ((spc->spc_flags & SPCF_NOINTR) != 0) 420 return 0; 421 /* 422 * Ensure that at least one CPU within the system 423 * is handing device interrupts. 424 */ 425 nintr = 0; 426 for (CPU_INFO_FOREACH(cii, ci2)) { 427 if ((ci2->ci_schedstate.spc_flags & SPCF_NOINTR) != 0) 428 continue; 429 if (ci2 == ci) 430 continue; 431 nintr++; 432 } 433 if (nintr == 0) 434 return EBUSY; 435 func = (xcfunc_t)cpu_xc_nointr; 436 } 437 438 where = xc_unicast(0, func, ci, NULL, ci); 439 xc_wait(where); 440 if (intr) { 441 KASSERT((spc->spc_flags & SPCF_NOINTR) == 0); 442 } else if ((spc->spc_flags & SPCF_NOINTR) == 0) { 443 /* If was not set offline, then it is busy */ 444 return EBUSY; 445 } 446 447 /* Direct interrupts away from the CPU and record the change. */ 448 cpu_intr_redistribute(); 449 spc->spc_lastmod = time_second; 450 return 0; 451 } 452 #else /* __HAVE_INTR_CONTROL */ 453 int 454 cpu_setintr(struct cpu_info *ci, bool intr) 455 { 456 457 return EOPNOTSUPP; 458 } 459 460 u_int 461 cpu_intr_count(struct cpu_info *ci) 462 { 463 464 return 0; /* 0 == "don't know" */ 465 } 466 #endif /* __HAVE_INTR_CONTROL */ 467 468 bool 469 cpu_softintr_p(void) 470 { 471 472 return (curlwp->l_pflag & LP_INTR) != 0; 473 } 474
Indexes created Sat Sep 27 14:09:57 GMT 2025