1 /* $NetBSD: subr_interrupt.c,v 1.7 2026/01/04 03:18:31 riastradh Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.7 2026/01/04 03:18:31 riastradh Exp $"); 31 32 #include <sys/param.h> 33 #include <sys/types.h> 34 35 #include <sys/conf.h> 36 #include <sys/cpu.h> 37 #include <sys/errno.h> 38 #include <sys/interrupt.h> 39 #include <sys/intr.h> 40 #include <sys/intrio.h> 41 #include <sys/kauth.h> 42 #include <sys/kcpuset.h> 43 #include <sys/kernel.h> 44 #include <sys/kmem.h> 45 #include <sys/proc.h> 46 #include <sys/sdt.h> 47 #include <sys/sysctl.h> 48 #include <sys/systm.h> 49 #include <sys/xcall.h> 50 51 #include <machine/limits.h> 52 53 #ifdef INTR_DEBUG 54 #define DPRINTF(msg) printf msg 55 #else 56 #define DPRINTF(msg) 57 #endif 58 59 static struct intrio_set kintrio_set = { "\0", NULL, 0 }; 60 61 #define UNSET_NOINTR_SHIELD 0 62 #define SET_NOINTR_SHIELD 1 63 64 static void 65 interrupt_shield_xcall(void *arg1, void *arg2) 66 { 67 struct cpu_info *ci; 68 struct schedstate_percpu *spc; 69 int s, shield; 70 71 ci = arg1; 72 shield = (int)(intptr_t)arg2; 73 spc = &ci->ci_schedstate; 74 75 s = splsched(); 76 if (shield == UNSET_NOINTR_SHIELD) 77 spc->spc_flags &= ~SPCF_NOINTR; 78 else if (shield == SET_NOINTR_SHIELD) 79 spc->spc_flags |= SPCF_NOINTR; 80 splx(s); 81 } 82 83 /* 84 * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags. 85 */ 86 static int 87 interrupt_shield(u_int cpu_idx, int shield) 88 { 89 struct cpu_info *ci; 90 struct schedstate_percpu *spc; 91 92 KASSERT(mutex_owned(&cpu_lock)); 93 94 ci = cpu_lookup(cpu_idx); 95 if (ci == NULL) 96 return SET_ERROR(EINVAL); 97 98 spc = &ci->ci_schedstate; 99 if (shield == UNSET_NOINTR_SHIELD) { 100 if ((spc->spc_flags & SPCF_NOINTR) == 0) 101 return 0; 102 } else if (shield == SET_NOINTR_SHIELD) { 103 if ((spc->spc_flags & SPCF_NOINTR) != 0) 104 return 0; 105 } 106 107 if (ci == curcpu() || !mp_online) { 108 interrupt_shield_xcall(ci, (void *)(intptr_t)shield); 109 } else { 110 uint64_t where; 111 where = xc_unicast(0, interrupt_shield_xcall, ci, 112 (void *)(intptr_t)shield, ci); 113 xc_wait(where); 114 } 115 116 spc->spc_lastmod = time_second; 117 return 0; 118 } 119 120 /* 121 * Move all assigned interrupts from "cpu_idx" to the other cpu as possible. 122 * The destination cpu is the lowest cpuid of available cpus. 123 * If there are no available cpus, give up to move interrupts. 124 */ 125 static int 126 interrupt_avert_intr(u_int cpu_idx) 127 { 128 kcpuset_t *cpuset; 129 struct intrids_handler *ii_handler; 130 intrid_t *ids; 131 int error = 0, i, nids; 132 133 kcpuset_create(&cpuset, true); 134 kcpuset_set(cpuset, cpu_idx); 135 136 ii_handler = interrupt_construct_intrids(cpuset); 137 if (ii_handler == NULL) { 138 error = SET_ERROR(EINVAL); 139 goto out; 140 } 141 nids = ii_handler->iih_nids; 142 if (nids == 0) { 143 error = 0; 144 goto destruct_out; 145 } 146 147 interrupt_get_available(cpuset); 148 kcpuset_clear(cpuset, cpu_idx); 149 if (kcpuset_iszero(cpuset)) { 150 DPRINTF(("%s: no available cpu\n", __func__)); 151 error = SET_ERROR(ENOENT); 152 goto destruct_out; 153 } 154 155 ids = ii_handler->iih_intrids; 156 for (i = 0; i < nids; i++) { 157 error = interrupt_distribute_handler(ids[i], cpuset, NULL); 158 if (error) 159 break; 160 } 161 162 destruct_out: 163 interrupt_destruct_intrids(ii_handler); 164 out: 165 kcpuset_destroy(cpuset); 166 return error; 167 } 168 169 /* 170 * Return actual intrio_list_line size. 171 * intrio_list_line size is variable by ncpu. 172 */ 173 static size_t 174 interrupt_intrio_list_line_size(void) 175 { 176 177 return sizeof(struct intrio_list_line) + 178 sizeof(struct intrio_list_line_cpu) * (ncpu - 1); 179 } 180 181 /* 182 * Return the size of interrupts list data on success. 183 * Reterun 0 on failed. 184 */ 185 static int 186 interrupt_intrio_list_size(size_t *ilsize) 187 { 188 struct intrids_handler *ii_handler; 189 190 *ilsize = 0; 191 192 /* buffer header */ 193 *ilsize += sizeof(struct intrio_list); 194 195 /* il_line body */ 196 ii_handler = interrupt_construct_intrids(kcpuset_running); 197 if (ii_handler == NULL) 198 return SET_ERROR(EOPNOTSUPP); 199 *ilsize += interrupt_intrio_list_line_size() * ii_handler->iih_nids; 200 201 interrupt_destruct_intrids(ii_handler); 202 return 0; 203 } 204 205 /* 206 * Set intrctl list data to "il", and return list structure bytes. 207 * If error occurred, return <0. 208 * If "data" == NULL, simply return list structure bytes. 209 */ 210 static int 211 interrupt_intrio_list(struct intrio_list *il, size_t ilsize) 212 { 213 struct intrio_list_line *illine; 214 kcpuset_t *assigned, *avail; 215 struct intrids_handler *ii_handler; 216 intrid_t *ids; 217 u_int cpu_idx; 218 int nids, intr_idx, error, line_size; 219 220 illine = (struct intrio_list_line *) 221 ((char *)il + sizeof(struct intrio_list)); 222 il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il); 223 224 kcpuset_create(&avail, true); 225 interrupt_get_available(avail); 226 kcpuset_create(&assigned, true); 227 228 ii_handler = interrupt_construct_intrids(kcpuset_running); 229 if (ii_handler == NULL) { 230 DPRINTF(("%s: interrupt_construct_intrids() failed\n", 231 __func__)); 232 error = SET_ERROR(EOPNOTSUPP); 233 goto out; 234 } 235 236 line_size = interrupt_intrio_list_line_size(); 237 /* ensure interrupts are not added after interrupt_intrio_list_size() */ 238 nids = ii_handler->iih_nids; 239 ids = ii_handler->iih_intrids; 240 if (ilsize < sizeof(struct intrio_list) + line_size * nids) { 241 DPRINTF(("%s: interrupts are added during execution.\n", 242 __func__)); 243 error = SET_ERROR(EAGAIN); 244 goto destruct_out; 245 } 246 247 for (intr_idx = 0; intr_idx < nids; intr_idx++) { 248 char devname[INTRDEVNAMEBUF]; 249 250 strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF); 251 interrupt_get_devname(ids[intr_idx], devname, sizeof(devname)); 252 strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF); 253 254 interrupt_get_assigned(ids[intr_idx], assigned); 255 for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) { 256 struct intrio_list_line_cpu *illcpu = 257 &illine->ill_cpu[cpu_idx]; 258 259 illcpu->illc_assigned = 260 kcpuset_isset(assigned, cpu_idx); 261 illcpu->illc_count = 262 interrupt_get_count(ids[intr_idx], cpu_idx); 263 } 264 265 illine = (struct intrio_list_line *) 266 ((char *)illine + line_size); 267 } 268 269 error = 0; 270 il->il_version = INTRIO_LIST_VERSION; 271 il->il_ncpus = ncpu; 272 il->il_nintrs = nids; 273 il->il_linesize = line_size; 274 il->il_bufsize = ilsize; 275 276 destruct_out: 277 interrupt_destruct_intrids(ii_handler); 278 out: 279 kcpuset_destroy(assigned); 280 kcpuset_destroy(avail); 281 282 return error; 283 } 284 285 /* 286 * "intrctl list" entry 287 */ 288 static int 289 interrupt_intrio_list_sysctl(SYSCTLFN_ARGS) 290 { 291 int error; 292 void *buf; 293 size_t ilsize; 294 295 if (oldlenp == NULL) 296 return SET_ERROR(EINVAL); 297 298 if ((error = interrupt_intrio_list_size(&ilsize)) != 0) 299 return error; 300 301 /* 302 * If oldp == NULL, the sysctl(8) caller process want to get the size of 303 * intrctl list data only. 304 */ 305 if (oldp == NULL) { 306 *oldlenp = ilsize; 307 return 0; 308 } 309 310 /* 311 * If oldp != NULL, the sysctl(8) caller process want to get both the 312 * size and the contents of intrctl list data. 313 */ 314 if (*oldlenp < ilsize) 315 return SET_ERROR(ENOMEM); 316 317 buf = kmem_zalloc(ilsize, KM_SLEEP); 318 if ((error = interrupt_intrio_list(buf, ilsize)) != 0) 319 goto out; 320 321 error = copyout(buf, oldp, ilsize); 322 out: 323 kmem_free(buf, ilsize); 324 return error; 325 } 326 327 /* 328 * "intrctl affinity" entry 329 */ 330 static int 331 interrupt_set_affinity_sysctl(SYSCTLFN_ARGS) 332 { 333 struct sysctlnode node; 334 struct intrio_set *iset; 335 cpuset_t *ucpuset; 336 kcpuset_t *kcpuset; 337 int error; 338 339 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR, 340 KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL); 341 if (error) 342 return SET_ERROR(EPERM); 343 344 node = *rnode; 345 iset = (struct intrio_set *)node.sysctl_data; 346 347 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 348 if (error != 0 || newp == NULL) 349 return error; 350 351 ucpuset = iset->cpuset; 352 kcpuset_create(&kcpuset, true); 353 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 354 if (error) 355 goto out; 356 if (kcpuset_iszero(kcpuset)) { 357 error = SET_ERROR(EINVAL); 358 goto out; 359 } 360 361 error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL); 362 363 out: 364 kcpuset_destroy(kcpuset); 365 return error; 366 } 367 368 /* 369 * "intrctl intr" entry 370 */ 371 static int 372 interrupt_intr_sysctl(SYSCTLFN_ARGS) 373 { 374 struct sysctlnode node; 375 struct intrio_set *iset; 376 cpuset_t *ucpuset; 377 kcpuset_t *kcpuset; 378 int error; 379 u_int cpu_idx; 380 381 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, 382 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); 383 if (error) 384 return SET_ERROR(EPERM); 385 386 node = *rnode; 387 iset = (struct intrio_set *)node.sysctl_data; 388 389 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 390 if (error != 0 || newp == NULL) 391 return error; 392 393 ucpuset = iset->cpuset; 394 kcpuset_create(&kcpuset, true); 395 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 396 if (error) 397 goto out; 398 if (kcpuset_iszero(kcpuset)) { 399 error = SET_ERROR(EINVAL); 400 goto out; 401 } 402 403 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ 404 405 mutex_enter(&cpu_lock); 406 error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD); 407 mutex_exit(&cpu_lock); 408 409 out: 410 kcpuset_destroy(kcpuset); 411 return error; 412 } 413 414 /* 415 * "intrctl nointr" entry 416 */ 417 static int 418 interrupt_nointr_sysctl(SYSCTLFN_ARGS) 419 { 420 struct sysctlnode node; 421 struct intrio_set *iset; 422 cpuset_t *ucpuset; 423 kcpuset_t *kcpuset; 424 int error; 425 u_int cpu_idx; 426 427 error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU, 428 KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL); 429 if (error) 430 return SET_ERROR(EPERM); 431 432 node = *rnode; 433 iset = (struct intrio_set *)node.sysctl_data; 434 435 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 436 if (error != 0 || newp == NULL) 437 return error; 438 439 ucpuset = iset->cpuset; 440 kcpuset_create(&kcpuset, true); 441 error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size); 442 if (error) 443 goto out; 444 if (kcpuset_iszero(kcpuset)) { 445 error = SET_ERROR(EINVAL); 446 goto out; 447 } 448 449 cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */ 450 451 mutex_enter(&cpu_lock); 452 error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD); 453 mutex_exit(&cpu_lock); 454 if (error) 455 goto out; 456 457 error = interrupt_avert_intr(cpu_idx); 458 459 out: 460 kcpuset_destroy(kcpuset); 461 return error; 462 } 463 464 SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup") 465 { 466 const struct sysctlnode *node = NULL; 467 468 sysctl_createv(clog, 0, NULL, &node, 469 CTLFLAG_PERMANENT, CTLTYPE_NODE, 470 "intr", SYSCTL_DESCR("Interrupt options"), 471 NULL, 0, NULL, 0, 472 CTL_KERN, CTL_CREATE, CTL_EOL); 473 474 sysctl_createv(clog, 0, &node, NULL, 475 CTLFLAG_PERMANENT, CTLTYPE_STRUCT, 476 "list", SYSCTL_DESCR("intrctl list"), 477 interrupt_intrio_list_sysctl, 0, NULL, 478 0, CTL_CREATE, CTL_EOL); 479 480 sysctl_createv(clog, 0, &node, NULL, 481 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 482 "affinity", SYSCTL_DESCR("set affinity"), 483 interrupt_set_affinity_sysctl, 0, &kintrio_set, 484 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 485 486 sysctl_createv(clog, 0, &node, NULL, 487 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 488 "intr", SYSCTL_DESCR("set intr"), 489 interrupt_intr_sysctl, 0, &kintrio_set, 490 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 491 492 sysctl_createv(clog, 0, &node, NULL, 493 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT, 494 "nointr", SYSCTL_DESCR("set nointr"), 495 interrupt_nointr_sysctl, 0, &kintrio_set, 496 sizeof(kintrio_set), CTL_CREATE, CTL_EOL); 497 } 498
Indexes created Sun Oct 25 00:26:45 UTC 2026