Home | History | Annotate | Line # | Download | only in kern
subr_interrupt.c revision 1.1 
      1 /*	$NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2015 Internet Initiative Japan Inc.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     18  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     20  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     21  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     22  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     23  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     24  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     25  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     26  * POSSIBILITY OF SUCH DAMAGE.
     27  */
     28 
     29 #include <sys/cdefs.h>
     30 __KERNEL_RCSID(0, "$NetBSD: subr_interrupt.c,v 1.1 2015/08/17 06:16:03 knakahara Exp $");
     31 
     32 #include <sys/param.h>
     33 #include <sys/systm.h>
     34 #include <sys/kernel.h>
     35 #include <sys/errno.h>
     36 #include <sys/cpu.h>
     37 #include <sys/interrupt.h>
     38 #include <sys/intr.h>
     39 #include <sys/kcpuset.h>
     40 #include <sys/kmem.h>
     41 #include <sys/proc.h>
     42 #include <sys/xcall.h>
     43 #include <sys/sysctl.h>
     44 
     45 #include <sys/conf.h>
     46 #include <sys/intrio.h>
     47 #include <sys/kauth.h>
     48 
     49 #include <machine/limits.h>
     50 
     51 #ifdef INTR_DEBUG
     52 #define DPRINTF(msg) printf msg
     53 #else
     54 #define DPRINTF(msg)
     55 #endif
     56 
     57 static struct intrio_set kintrio_set = { "\0", NULL, 0 };
     58 
     59 #define UNSET_NOINTR_SHIELD	0
     60 #define SET_NOINTR_SHIELD	1
     61 
     62 static void
     63 interrupt_shield_xcall(void *arg1, void *arg2)
     64 {
     65 	struct cpu_info *ci;
     66 	struct schedstate_percpu *spc;
     67 	int s, shield;
     68 
     69 	ci = arg1;
     70 	shield = (int)(intptr_t)arg2;
     71 	spc = &ci->ci_schedstate;
     72 
     73 	s = splsched();
     74 	if (shield == UNSET_NOINTR_SHIELD)
     75 		spc->spc_flags &= ~SPCF_NOINTR;
     76 	else if (shield == SET_NOINTR_SHIELD)
     77 		spc->spc_flags |= SPCF_NOINTR;
     78 	splx(s);
     79 }
     80 
     81 /*
     82  * Change SPCF_NOINTR flag of schedstate_percpu->spc_flags.
     83  */
     84 static int
     85 interrupt_shield(u_int cpu_idx, int shield)
     86 {
     87 	struct cpu_info *ci;
     88 	struct schedstate_percpu *spc;
     89 
     90 	KASSERT(mutex_owned(&cpu_lock));
     91 
     92 	ci = cpu_lookup(cpu_idx);
     93 	if (ci == NULL)
     94 		return EINVAL;
     95 
     96 	spc = &ci->ci_schedstate;
     97 	if (shield == UNSET_NOINTR_SHIELD) {
     98 		if ((spc->spc_flags & SPCF_NOINTR) == 0)
     99 			return 0;
    100 	} else if (shield == SET_NOINTR_SHIELD) {
    101 		if ((spc->spc_flags & SPCF_NOINTR) != 0)
    102 			return 0;
    103 	}
    104 
    105 	if (ci == curcpu() || !mp_online) {
    106 		interrupt_shield_xcall(ci, (void *)(intptr_t)shield);
    107 	} else {
    108 		uint64_t where;
    109 		where = xc_unicast(0, interrupt_shield_xcall, ci,
    110 			(void *)(intptr_t)shield, ci);
    111 		xc_wait(where);
    112 	}
    113 
    114 	spc->spc_lastmod = time_second;
    115 	return 0;
    116 }
    117 
    118 /*
    119  * Move all assigned interrupts from "cpu_idx" to the other cpu as possible.
    120  * The destination cpu is the lowest cpuid of available cpus.
    121  * If there are no available cpus, give up to move interrupts.
    122  */
    123 static int
    124 interrupt_avert_intr(u_int cpu_idx)
    125 {
    126 	kcpuset_t *cpuset;
    127 	struct intrids_handler *ii_handler;
    128 	intrid_t *ids;
    129 	int error, i, nids;
    130 
    131 	kcpuset_create(&cpuset, true);
    132 	kcpuset_set(cpuset, cpu_idx);
    133 
    134 	ii_handler = interrupt_construct_intrids(cpuset);
    135 	if (ii_handler == NULL) {
    136 		error = ENOMEM;
    137 		goto out;
    138 	}
    139 	nids = ii_handler->iih_nids;
    140 	if (nids == 0) {
    141 		error = 0;
    142 		goto destruct_out;
    143 	}
    144 
    145 	interrupt_get_available(cpuset);
    146 	kcpuset_clear(cpuset, cpu_idx);
    147 	if (kcpuset_iszero(cpuset)) {
    148 		DPRINTF(("%s: no available cpu\n", __func__));
    149 		error = ENOENT;
    150 		goto destruct_out;
    151 	}
    152 
    153 	ids = ii_handler->iih_intrids;
    154 	for (i = 0; i < nids; i++) {
    155 		error = interrupt_distribute_handler(ids[i], cpuset, NULL);
    156 		if (error)
    157 			break;
    158 	}
    159 
    160  destruct_out:
    161 	interrupt_destruct_intrids(ii_handler);
    162  out:
    163 	kcpuset_destroy(cpuset);
    164 	return error;
    165 }
    166 
    167 /*
    168  * Return actual intrio_list_line size.
    169  * intrio_list_line size is variable by ncpu.
    170  */
    171 static size_t
    172 interrupt_intrio_list_line_size(void)
    173 {
    174 
    175 	return sizeof(struct intrio_list_line) +
    176 		sizeof(struct intrio_list_line_cpu) * (ncpu - 1);
    177 }
    178 
    179 /*
    180  * Return the size of interrupts list data on success.
    181  * Reterun 0 on failed.
    182  */
    183 static size_t
    184 interrupt_intrio_list_size(void)
    185 {
    186 	struct intrids_handler *ii_handler;
    187 	size_t ilsize;
    188 
    189 	ilsize = 0;
    190 
    191 	/* buffer header */
    192 	ilsize += sizeof(struct intrio_list);
    193 
    194 	/* il_line body */
    195 	ii_handler = interrupt_construct_intrids(kcpuset_running);
    196 	if (ii_handler == NULL)
    197 		return 0;
    198 	ilsize += interrupt_intrio_list_line_size() * (ii_handler->iih_nids);
    199 
    200 	interrupt_destruct_intrids(ii_handler);
    201 	return ilsize;
    202 }
    203 
    204 /*
    205  * Set intrctl list data to "il", and return list structure bytes.
    206  * If error occured, return <0.
    207  * If "data" == NULL, simply return list structure bytes.
    208  */
    209 static int
    210 interrupt_intrio_list(struct intrio_list *il, int length)
    211 {
    212 	struct intrio_list_line *illine;
    213 	kcpuset_t *assigned, *avail;
    214 	struct intrids_handler *ii_handler;
    215 	intrid_t *ids;
    216 	size_t ilsize;
    217 	u_int cpu_idx;
    218 	int nids, intr_idx, ret, line_size;
    219 
    220 	ilsize = interrupt_intrio_list_size();
    221 	if (ilsize == 0)
    222 		return -ENOMEM;
    223 
    224 	if (il == NULL)
    225 		return ilsize;
    226 
    227 	if (length < ilsize)
    228 		return -ENOMEM;
    229 
    230 	illine = (struct intrio_list_line *)
    231 		((char *)il + sizeof(struct intrio_list));
    232 	il->il_lineoffset = (off_t)((uintptr_t)illine - (uintptr_t)il);
    233 
    234 	kcpuset_create(&avail, true);
    235 	interrupt_get_available(avail);
    236 	kcpuset_create(&assigned, true);
    237 
    238 	ii_handler = interrupt_construct_intrids(kcpuset_running);
    239 	if (ii_handler == NULL) {
    240 		DPRINTF(("%s: interrupt_construct_intrids() failed\n",
    241 			__func__));
    242 		ret = -ENOMEM;
    243 		goto out;
    244 	}
    245 
    246 	line_size = interrupt_intrio_list_line_size();
    247 	/* ensure interrupts are not added after interrupt_intrio_list_size(). */
    248 	nids = ii_handler->iih_nids;
    249 	ids = ii_handler->iih_intrids;
    250 	if (ilsize < sizeof(struct intrio_list) + line_size * nids) {
    251 		DPRINTF(("%s: interrupts are added during execution.\n",
    252 			__func__));
    253 		ret = -ENOMEM;
    254 		goto destruct_out;
    255 	}
    256 
    257 	for (intr_idx = 0; intr_idx < nids; intr_idx++) {
    258 		char devname[INTRDEVNAMEBUF];
    259 
    260 		strncpy(illine->ill_intrid, ids[intr_idx], INTRIDBUF);
    261 		interrupt_get_devname(ids[intr_idx], devname, sizeof(devname));
    262 		strncpy(illine->ill_xname, devname, INTRDEVNAMEBUF);
    263 
    264 		interrupt_get_assigned(ids[intr_idx], assigned);
    265 		for (cpu_idx = 0; cpu_idx < ncpu; cpu_idx++) {
    266 			struct intrio_list_line_cpu *illcpu =
    267 				&illine->ill_cpu[cpu_idx];
    268 
    269 			illcpu->illc_assigned =
    270 				kcpuset_isset(assigned, cpu_idx) ? true : false;
    271 			illcpu->illc_count =
    272 				interrupt_get_count(ids[intr_idx], cpu_idx);
    273 		}
    274 
    275 		illine = (struct intrio_list_line *)
    276 			((char *)illine + line_size);
    277 	}
    278 
    279 	ret = ilsize;
    280 	il->il_version = INTRIO_LIST_VERSION;
    281 	il->il_ncpus = ncpu;
    282 	il->il_nintrs = nids;
    283 	il->il_linesize = line_size;
    284 	il->il_bufsize = ilsize;
    285 
    286  destruct_out:
    287 	interrupt_destruct_intrids(ii_handler);
    288  out:
    289 	kcpuset_destroy(assigned);
    290 	kcpuset_destroy(avail);
    291 
    292 	return ret;
    293 }
    294 
    295 /*
    296  * "intrctl list" entry
    297  */
    298 static int
    299 interrupt_intrio_list_sysctl(SYSCTLFN_ARGS)
    300 {
    301 	int ret, error;
    302 	void *buf;
    303 
    304 	if (oldlenp == NULL)
    305 		return EINVAL;
    306 
    307 	/*
    308 	 * If oldp == NULL, the sysctl(8) caller process want to get the size of
    309 	 * intrctl list data only.
    310 	 */
    311 	if (oldp == NULL) {
    312 		ret = interrupt_intrio_list(NULL, 0);
    313 		if (ret < 0)
    314 			return -ret;
    315 
    316 		*oldlenp = ret;
    317 		return 0;
    318 	}
    319 
    320 	/*
    321 	 * If oldp != NULL, the sysctl(8) caller process want to get both the size
    322 	 * and the contents of intrctl list data.
    323 	 */
    324 	if (*oldlenp == 0)
    325 		return ENOMEM;
    326 
    327 	buf = kmem_zalloc(*oldlenp, KM_SLEEP);
    328 	if (buf == NULL)
    329 		return ENOMEM;
    330 
    331 	ret = interrupt_intrio_list(buf, *oldlenp);
    332 	if (ret < 0) {
    333 		error = -ret;
    334 		goto out;
    335 	}
    336 	error = copyout(buf, oldp, *oldlenp);
    337 
    338  out:
    339 	kmem_free(buf, *oldlenp);
    340 	return error;
    341 }
    342 
    343 /*
    344  * "intrctl affinity" entry
    345  */
    346 static int
    347 interrupt_set_affinity_sysctl(SYSCTLFN_ARGS)
    348 {
    349 	struct sysctlnode node;
    350 	struct intrio_set *iset;
    351 	cpuset_t *ucpuset;
    352 	kcpuset_t *kcpuset;
    353 	int error;
    354 
    355 	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_INTR,
    356 	    KAUTH_REQ_SYSTEM_INTR_AFFINITY, NULL, NULL, NULL);
    357 	if (error)
    358 		return EPERM;
    359 
    360 	node = *rnode;
    361 	iset = (struct intrio_set *)node.sysctl_data;
    362 
    363 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    364 	if (error != 0 || newp == NULL)
    365 		return error;
    366 
    367 	ucpuset = iset->cpuset;
    368 	kcpuset_create(&kcpuset, true);
    369 	error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
    370 	if (error)
    371 		goto out;
    372 	if (kcpuset_iszero(kcpuset)) {
    373 		error = EINVAL;
    374 		goto out;
    375 	}
    376 
    377 	error = interrupt_distribute_handler(iset->intrid, kcpuset, NULL);
    378 
    379  out:
    380 	kcpuset_destroy(kcpuset);
    381 	return error;
    382 }
    383 
    384 /*
    385  * "intrctl intr" entry
    386  */
    387 static int
    388 interrupt_intr_sysctl(SYSCTLFN_ARGS)
    389 {
    390 	struct sysctlnode node;
    391 	struct intrio_set *iset;
    392 	cpuset_t *ucpuset;
    393 	kcpuset_t *kcpuset;
    394 	int error;
    395 	u_int cpu_idx;
    396 
    397 	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
    398 	    KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
    399 	if (error)
    400 		return EPERM;
    401 
    402 	node = *rnode;
    403 	iset = (struct intrio_set *)node.sysctl_data;
    404 
    405 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    406 	if (error != 0 || newp == NULL)
    407 		return error;
    408 
    409 	ucpuset = iset->cpuset;
    410 	kcpuset_create(&kcpuset, true);
    411 	error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
    412 	if (error)
    413 		goto out;
    414 	if (kcpuset_iszero(kcpuset)) {
    415 		error = EINVAL;
    416 		goto out;
    417 	}
    418 
    419 	cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
    420 
    421 	mutex_enter(&cpu_lock);
    422 	error = interrupt_shield(cpu_idx, UNSET_NOINTR_SHIELD);
    423 	mutex_exit(&cpu_lock);
    424 
    425  out:
    426 	kcpuset_destroy(kcpuset);
    427 	return error;
    428 }
    429 
    430 /*
    431  * "intrctl nointr" entry
    432  */
    433 static int
    434 interrupt_nointr_sysctl(SYSCTLFN_ARGS)
    435 {
    436 	struct sysctlnode node;
    437 	struct intrio_set *iset;
    438 	cpuset_t *ucpuset;
    439 	kcpuset_t *kcpuset;
    440 	int error;
    441 	u_int cpu_idx;
    442 
    443 	error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_CPU,
    444 	    KAUTH_REQ_SYSTEM_CPU_SETSTATE, NULL, NULL, NULL);
    445 	if (error)
    446 		return EPERM;
    447 
    448 	node = *rnode;
    449 	iset = (struct intrio_set *)node.sysctl_data;
    450 
    451 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    452 	if (error != 0 || newp == NULL)
    453 		return error;
    454 
    455 	ucpuset = iset->cpuset;
    456 	kcpuset_create(&kcpuset, true);
    457 	error = kcpuset_copyin(ucpuset, kcpuset, iset->cpuset_size);
    458 	if (error)
    459 		goto out;
    460 	if (kcpuset_iszero(kcpuset)) {
    461 		error = EINVAL;
    462 		goto out;
    463 	}
    464 
    465 	cpu_idx = kcpuset_ffs(kcpuset) - 1; /* support one CPU only */
    466 
    467 	mutex_enter(&cpu_lock);
    468 	error = interrupt_shield(cpu_idx, SET_NOINTR_SHIELD);
    469 	mutex_exit(&cpu_lock);
    470 	if (error)
    471 		goto out;
    472 
    473 	error = interrupt_avert_intr(cpu_idx);
    474 
    475  out:
    476 	kcpuset_destroy(kcpuset);
    477 	return error;
    478 }
    479 
    480 SYSCTL_SETUP(sysctl_interrupt_setup, "sysctl interrupt setup")
    481 {
    482 	const struct sysctlnode *node = NULL;
    483 
    484 	sysctl_createv(clog, 0, NULL, &node,
    485 		       CTLFLAG_PERMANENT, CTLTYPE_NODE,
    486 		       "intr", SYSCTL_DESCR("Interrupt options"),
    487 		       NULL, 0, NULL, 0,
    488 		       CTL_KERN, CTL_CREATE, CTL_EOL);
    489 
    490 	sysctl_createv(clog, 0, &node, NULL,
    491 		       CTLFLAG_PERMANENT, CTLTYPE_STRUCT,
    492 		       "list", SYSCTL_DESCR("intrctl list"),
    493 		       interrupt_intrio_list_sysctl, 0, NULL,
    494 		        0, CTL_CREATE, CTL_EOL);
    495 
    496 	sysctl_createv(clog, 0, &node, NULL,
    497 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
    498 		       "affinity", SYSCTL_DESCR("set affinity"),
    499 		       interrupt_set_affinity_sysctl, 0, &kintrio_set,
    500 		       sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
    501 
    502 	sysctl_createv(clog, 0, &node, NULL,
    503 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
    504 		       "intr", SYSCTL_DESCR("set intr"),
    505 		       interrupt_intr_sysctl, 0, &kintrio_set,
    506 		       sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
    507 
    508 	sysctl_createv(clog, 0, &node, NULL,
    509 		       CTLFLAG_PERMANENT|CTLFLAG_READWRITE, CTLTYPE_STRUCT,
    510 		       "nointr", SYSCTL_DESCR("set nointr"),
    511 		       interrupt_nointr_sysctl, 0, &kintrio_set,
    512 		       sizeof(kintrio_set), CTL_CREATE, CTL_EOL);
    513 }
    514