Home | History | Annotate | Line # | Download | only in kern
kern_softint.c revision 1.75
      1 /*	$NetBSD: kern_softint.c,v 1.75 2023/08/04 12:24:36 riastradh Exp $	*/
      2 
      3 /*-
      4  * Copyright (c) 2007, 2008, 2019, 2020 The NetBSD Foundation, Inc.
      5  * All rights reserved.
      6  *
      7  * This code is derived from software contributed to The NetBSD Foundation
      8  * by Andrew Doran.
      9  *
     10  * Redistribution and use in source and binary forms, with or without
     11  * modification, are permitted provided that the following conditions
     12  * are met:
     13  * 1. Redistributions of source code must retain the above copyright
     14  *    notice, this list of conditions and the following disclaimer.
     15  * 2. Redistributions in binary form must reproduce the above copyright
     16  *    notice, this list of conditions and the following disclaimer in the
     17  *    documentation and/or other materials provided with the distribution.
     18  *
     19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29  * POSSIBILITY OF SUCH DAMAGE.
     30  */
     31 
     32 /*
     33  * Generic software interrupt framework.
     34  *
     35  * Overview
     36  *
     37  *	The soft interrupt framework provides a mechanism to schedule a
     38  *	low priority callback that runs with thread context.  It allows
     39  *	for dynamic registration of software interrupts, and for fair
     40  *	queueing and prioritization of those interrupts.  The callbacks
     41  *	can be scheduled to run from nearly any point in the kernel: by
     42  *	code running with thread context, by code running from a
     43  *	hardware interrupt handler, and at any interrupt priority
     44  *	level.
     45  *
     46  * Priority levels
     47  *
     48  *	Since soft interrupt dispatch can be tied to the underlying
     49  *	architecture's interrupt dispatch code, it can be limited
     50  *	both by the capabilities of the hardware and the capabilities
     51  *	of the interrupt dispatch code itself.  The number of priority
     52  *	levels is restricted to four.  In order of priority (lowest to
     53  *	highest) the levels are: clock, bio, net, serial.
     54  *
     55  *	The names are symbolic and in isolation do not have any direct
     56  *	connection with a particular kind of device activity: they are
     57  *	only meant as a guide.
     58  *
     59  *	The four priority levels map directly to scheduler priority
     60  *	levels, and where the architecture implements 'fast' software
     61  *	interrupts, they also map onto interrupt priorities.  The
     62  *	interrupt priorities are intended to be hidden from machine
     63  *	independent code, which should use thread-safe mechanisms to
     64  *	synchronize with software interrupts (for example: mutexes).
     65  *
     66  * Capabilities
     67  *
     68  *	Software interrupts run with limited machine context.  In
     69  *	particular, they do not posess any address space context.  They
     70  *	should not try to operate on user space addresses, or to use
     71  *	virtual memory facilities other than those noted as interrupt
     72  *	safe.
     73  *
     74  *	Unlike hardware interrupts, software interrupts do have thread
     75  *	context.  They may block on synchronization objects, sleep, and
     76  *	resume execution at a later time.
     77  *
     78  *	Since software interrupts are a limited resource and run with
     79  *	higher priority than most other LWPs in the system, all
     80  *	block-and-resume activity by a software interrupt must be kept
     81  *	short to allow further processing at that level to continue.  By
     82  *	extension, code running with process context must take care to
     83  *	ensure that any lock that may be taken from a software interrupt
     84  *	can not be held for more than a short period of time.
     85  *
     86  *	The kernel does not allow software interrupts to use facilities
     87  *	or perform actions that may block for a significant amount of
     88  *	time.  This means that it's not valid for a software interrupt
     89  *	to sleep on condition variables	or wait for resources to become
     90  *	available (for example,	memory).
     91  *
     92  * Per-CPU operation
     93  *
     94  *	If a soft interrupt is triggered on a CPU, it can only be
     95  *	dispatched on the same CPU.  Each LWP dedicated to handling a
     96  *	soft interrupt is bound to its home CPU, so if the LWP blocks
     97  *	and needs to run again, it can only run there.  Nearly all data
     98  *	structures used to manage software interrupts are per-CPU.
     99  *
    100  *	The per-CPU requirement is intended to reduce "ping-pong" of
    101  *	cache lines between CPUs: lines occupied by data structures
    102  *	used to manage the soft interrupts, and lines occupied by data
    103  *	items being passed down to the soft interrupt.  As a positive
    104  *	side effect, this also means that the soft interrupt dispatch
    105  *	code does not need to to use spinlocks to synchronize.
    106  *
    107  * Generic implementation
    108  *
    109  *	A generic, low performance implementation is provided that
    110  *	works across all architectures, with no machine-dependent
    111  *	modifications needed.  This implementation uses the scheduler,
    112  *	and so has a number of restrictions:
    113  *
    114  *	1) The software interrupts are not currently preemptive, so
    115  *	must wait for the currently executing LWP to yield the CPU.
    116  *	This can introduce latency.
    117  *
    118  *	2) An expensive context switch is required for a software
    119  *	interrupt to be handled.
    120  *
    121  * 'Fast' software interrupts
    122  *
    123  *	If an architectures defines __HAVE_FAST_SOFTINTS, it implements
    124  *	the fast mechanism.  Threads running either in the kernel or in
    125  *	userspace will be interrupted, but will not be preempted.  When
    126  *	the soft interrupt completes execution, the interrupted LWP
    127  *	is resumed.  Interrupt dispatch code must provide the minimum
    128  *	level of context necessary for the soft interrupt to block and
    129  *	be resumed at a later time.  The machine-dependent dispatch
    130  *	path looks something like the following:
    131  *
    132  *	softintr()
    133  *	{
    134  *		go to IPL_HIGH if necessary for switch;
    135  *		save any necessary registers in a format that can be
    136  *		    restored by cpu_switchto if the softint blocks;
    137  *		arrange for cpu_switchto() to restore into the
    138  *		    trampoline function;
    139  *		identify LWP to handle this interrupt;
    140  *		switch to the LWP's stack;
    141  *		switch register stacks, if necessary;
    142  *		assign new value of curlwp;
    143  *		call MI softint_dispatch, passing old curlwp and IPL
    144  *		    to execute interrupt at;
    145  *		switch back to old stack;
    146  *		switch back to old register stack, if necessary;
    147  *		restore curlwp;
    148  *		return to interrupted LWP;
    149  *	}
    150  *
    151  *	If the soft interrupt blocks, a trampoline function is returned
    152  *	to in the context of the interrupted LWP, as arranged for by
    153  *	softint():
    154  *
    155  *	softint_ret()
    156  *	{
    157  *		unlock soft interrupt LWP;
    158  *		resume interrupt processing, likely returning to
    159  *		    interrupted LWP or dispatching another, different
    160  *		    interrupt;
    161  *	}
    162  *
    163  *	Once the soft interrupt has fired (and even if it has blocked),
    164  *	no further soft interrupts at that level will be triggered by
    165  *	MI code until the soft interrupt handler has ceased execution.
    166  *	If a soft interrupt handler blocks and is resumed, it resumes
    167  *	execution as a normal LWP (kthread) and gains VM context.  Only
    168  *	when it has completed and is ready to fire again will it
    169  *	interrupt other threads.
    170  */
    171 
    172 #include <sys/cdefs.h>
    173 __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.75 2023/08/04 12:24:36 riastradh Exp $");
    174 
    175 #include <sys/param.h>
    176 #include <sys/proc.h>
    177 #include <sys/intr.h>
    178 #include <sys/ipi.h>
    179 #include <sys/lock.h>
    180 #include <sys/mutex.h>
    181 #include <sys/kernel.h>
    182 #include <sys/kthread.h>
    183 #include <sys/evcnt.h>
    184 #include <sys/cpu.h>
    185 #include <sys/xcall.h>
    186 #include <sys/psref.h>
    187 #include <sys/sdt.h>
    188 
    189 #include <uvm/uvm_extern.h>
    190 
    191 /* This could overlap with signal info in struct lwp. */
    192 typedef struct softint {
    193 	SIMPLEQ_HEAD(, softhand) si_q;
    194 	struct lwp		*si_lwp;
    195 	struct cpu_info		*si_cpu;
    196 	uintptr_t		si_machdep;
    197 	struct evcnt		si_evcnt;
    198 	struct evcnt		si_evcnt_block;
    199 	volatile int		si_active;
    200 	int			si_ipl;
    201 	char			si_name[8];
    202 	char			si_name_block[8+6];
    203 } softint_t;
    204 
    205 typedef struct softhand {
    206 	SIMPLEQ_ENTRY(softhand)	sh_q;
    207 	void			(*sh_func)(void *);
    208 	void			*sh_arg;
    209 	softint_t		*sh_isr;
    210 	u_int			sh_flags;
    211 	u_int			sh_ipi_id;
    212 } softhand_t;
    213 
    214 typedef struct softcpu {
    215 	struct cpu_info		*sc_cpu;
    216 	softint_t		sc_int[SOFTINT_COUNT];
    217 	softhand_t		sc_hand[1];
    218 } softcpu_t;
    219 
    220 static void	softint_thread(void *);
    221 
    222 u_int		softint_bytes = 32768;
    223 u_int		softint_timing;
    224 static u_int	softint_max;
    225 static kmutex_t	softint_lock;
    226 
    227 SDT_PROBE_DEFINE4(sdt, kernel, softint, establish,
    228     "void *"/*sih*/,
    229     "void (*)(void *)"/*func*/,
    230     "void *"/*arg*/,
    231     "unsigned"/*flags*/);
    232 
    233 SDT_PROBE_DEFINE1(sdt, kernel, softint, disestablish,
    234     "void *"/*sih*/);
    235 
    236 SDT_PROBE_DEFINE2(sdt, kernel, softint, schedule,
    237     "void *"/*sih*/,
    238     "struct cpu_info *"/*ci*/);
    239 
    240 SDT_PROBE_DEFINE4(sdt, kernel, softint, entry,
    241     "void *"/*sih*/,
    242     "void (*)(void *)"/*func*/,
    243     "void *"/*arg*/,
    244     "unsigned"/*flags*/);
    245 
    246 SDT_PROBE_DEFINE4(sdt, kernel, softint, return,
    247     "void *"/*sih*/,
    248     "void (*)(void *)"/*func*/,
    249     "void *"/*arg*/,
    250     "unsigned"/*flags*/);
    251 
    252 /*
    253  * softint_init_isr:
    254  *
    255  *	Initialize a single interrupt level for a single CPU.
    256  */
    257 static void
    258 softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level,
    259     int ipl)
    260 {
    261 	struct cpu_info *ci;
    262 	softint_t *si;
    263 	int error;
    264 
    265 	si = &sc->sc_int[level];
    266 	ci = sc->sc_cpu;
    267 	si->si_cpu = ci;
    268 
    269 	SIMPLEQ_INIT(&si->si_q);
    270 
    271 	error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR |
    272 	    KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp,
    273 	    "soft%s/%u", desc, ci->ci_index);
    274 	if (error != 0)
    275 		panic("softint_init_isr: error %d", error);
    276 
    277 	snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc,
    278 	    ci->ci_index);
    279 	evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL,
    280 	   "softint", si->si_name);
    281 	snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u",
    282 	    desc, ci->ci_index);
    283 	evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL,
    284 	   "softint", si->si_name_block);
    285 
    286 	si->si_ipl = ipl;
    287 	si->si_lwp->l_private = si;
    288 	softint_init_md(si->si_lwp, level, &si->si_machdep);
    289 }
    290 
    291 /*
    292  * softint_init:
    293  *
    294  *	Initialize per-CPU data structures.  Called from mi_cpu_attach().
    295  */
    296 void
    297 softint_init(struct cpu_info *ci)
    298 {
    299 	static struct cpu_info *first;
    300 	softcpu_t *sc, *scfirst;
    301 	softhand_t *sh, *shmax;
    302 
    303 	if (first == NULL) {
    304 		/* Boot CPU. */
    305 		first = ci;
    306 		mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE);
    307 		softint_bytes = round_page(softint_bytes);
    308 		softint_max = (softint_bytes - sizeof(softcpu_t)) /
    309 		    sizeof(softhand_t);
    310 	}
    311 
    312 	/* Use uvm_km(9) for persistent, page-aligned allocation. */
    313 	sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0,
    314 	    UVM_KMF_WIRED | UVM_KMF_ZERO);
    315 	if (sc == NULL)
    316 		panic("softint_init_cpu: cannot allocate memory");
    317 
    318 	ci->ci_data.cpu_softcpu = sc;
    319 	ci->ci_data.cpu_softints = 0;
    320 	sc->sc_cpu = ci;
    321 
    322 	softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET,
    323 	    IPL_SOFTNET);
    324 	softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO,
    325 	    IPL_SOFTBIO);
    326 	softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK,
    327 	    IPL_SOFTCLOCK);
    328 	softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL,
    329 	    IPL_SOFTSERIAL);
    330 
    331 	if (first != ci) {
    332 		mutex_enter(&softint_lock);
    333 		scfirst = first->ci_data.cpu_softcpu;
    334 		sh = sc->sc_hand;
    335 		memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max);
    336 		/* Update pointers for this CPU. */
    337 		for (shmax = sh + softint_max; sh < shmax; sh++) {
    338 			if (sh->sh_func == NULL)
    339 				continue;
    340 			sh->sh_isr =
    341 			    &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK];
    342 		}
    343 		mutex_exit(&softint_lock);
    344 	}
    345 }
    346 
    347 /*
    348  * softint_establish:
    349  *
    350  *	Register a software interrupt handler.
    351  */
    352 void *
    353 softint_establish(u_int flags, void (*func)(void *), void *arg)
    354 {
    355 	CPU_INFO_ITERATOR cii;
    356 	struct cpu_info *ci;
    357 	softcpu_t *sc;
    358 	softhand_t *sh;
    359 	u_int level, index;
    360 	u_int ipi_id = 0;
    361 	void *sih;
    362 
    363 	level = (flags & SOFTINT_LVLMASK);
    364 	KASSERT(level < SOFTINT_COUNT);
    365 	KASSERT((flags & SOFTINT_IMPMASK) == 0);
    366 
    367 	mutex_enter(&softint_lock);
    368 
    369 	/* Find a free slot. */
    370 	sc = curcpu()->ci_data.cpu_softcpu;
    371 	for (index = 1; index < softint_max; index++) {
    372 		if (sc->sc_hand[index].sh_func == NULL)
    373 			break;
    374 	}
    375 	if (index == softint_max) {
    376 		mutex_exit(&softint_lock);
    377 		printf("WARNING: softint_establish: table full, "
    378 		    "increase softint_bytes\n");
    379 		return NULL;
    380 	}
    381 	sih = (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc);
    382 
    383 	if (flags & SOFTINT_RCPU) {
    384 		if ((ipi_id = ipi_register(softint_schedule, sih)) == 0) {
    385 			mutex_exit(&softint_lock);
    386 			return NULL;
    387 		}
    388 	}
    389 
    390 	/* Set up the handler on each CPU. */
    391 	if (ncpu < 2) {
    392 		/* XXX hack for machines with no CPU_INFO_FOREACH() early on */
    393 		sc = curcpu()->ci_data.cpu_softcpu;
    394 		sh = &sc->sc_hand[index];
    395 		sh->sh_isr = &sc->sc_int[level];
    396 		sh->sh_func = func;
    397 		sh->sh_arg = arg;
    398 		sh->sh_flags = flags;
    399 		sh->sh_ipi_id = ipi_id;
    400 	} else for (CPU_INFO_FOREACH(cii, ci)) {
    401 		sc = ci->ci_data.cpu_softcpu;
    402 		sh = &sc->sc_hand[index];
    403 		sh->sh_isr = &sc->sc_int[level];
    404 		sh->sh_func = func;
    405 		sh->sh_arg = arg;
    406 		sh->sh_flags = flags;
    407 		sh->sh_ipi_id = ipi_id;
    408 	}
    409 	mutex_exit(&softint_lock);
    410 
    411 	SDT_PROBE4(sdt, kernel, softint, establish,  sih, func, arg, flags);
    412 
    413 	return sih;
    414 }
    415 
    416 /*
    417  * softint_disestablish:
    418  *
    419  *	Unregister a software interrupt handler.  The soft interrupt could
    420  *	still be active at this point, but the caller commits not to try
    421  *	and trigger it again once this call is made.  The caller must not
    422  *	hold any locks that could be taken from soft interrupt context,
    423  *	because we will wait for the softint to complete if it's still
    424  *	running.
    425  */
    426 void
    427 softint_disestablish(void *arg)
    428 {
    429 	CPU_INFO_ITERATOR cii;
    430 	struct cpu_info *ci;
    431 	softcpu_t *sc;
    432 	softhand_t *sh;
    433 	uintptr_t offset;
    434 
    435 	offset = (uintptr_t)arg;
    436 	KASSERT(offset != 0);
    437 	KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u",
    438 	    offset, softint_bytes);
    439 
    440 	/*
    441 	 * Unregister IPI handler if there is any.  Note: there is no need
    442 	 * to disable preemption here - ID is stable.
    443 	 */
    444 	sc = curcpu()->ci_data.cpu_softcpu;
    445 	sh = (softhand_t *)((uint8_t *)sc + offset);
    446 	if (sh->sh_ipi_id) {
    447 		ipi_unregister(sh->sh_ipi_id);
    448 	}
    449 
    450 	/*
    451 	 * Run a dummy softint at the same level on all CPUs and wait for
    452 	 * completion, to make sure this softint is no longer running
    453 	 * anywhere.
    454 	 */
    455 	xc_barrier(XC_HIGHPRI_IPL(sh->sh_isr->si_ipl));
    456 
    457 	/*
    458 	 * Notify dtrace probe when the old softint can't be running
    459 	 * any more, but before it can be recycled for a new softint.
    460 	 */
    461 	SDT_PROBE1(sdt, kernel, softint, disestablish,  arg);
    462 
    463 	/* Clear the handler on each CPU. */
    464 	mutex_enter(&softint_lock);
    465 	for (CPU_INFO_FOREACH(cii, ci)) {
    466 		sc = ci->ci_data.cpu_softcpu;
    467 		sh = (softhand_t *)((uint8_t *)sc + offset);
    468 		KASSERT(sh->sh_func != NULL);
    469 		sh->sh_func = NULL;
    470 	}
    471 	mutex_exit(&softint_lock);
    472 }
    473 
    474 /*
    475  * softint_schedule:
    476  *
    477  *	Trigger a software interrupt.  Must be called from a hardware
    478  *	interrupt handler, or with preemption disabled (since we are
    479  *	using the value of curcpu()).
    480  */
    481 void
    482 softint_schedule(void *arg)
    483 {
    484 	softhand_t *sh;
    485 	softint_t *si;
    486 	uintptr_t offset;
    487 	int s;
    488 
    489 	SDT_PROBE2(sdt, kernel, softint, schedule,  arg, /*ci*/NULL);
    490 
    491 	/*
    492 	 * If this assert fires, rather than disabling preemption explicitly
    493 	 * to make it stop, consider that you are probably using a softint
    494 	 * when you don't need to.
    495 	 */
    496 	KASSERT(kpreempt_disabled());
    497 
    498 	/* Find the handler record for this CPU. */
    499 	offset = (uintptr_t)arg;
    500 	KASSERT(offset != 0);
    501 	KASSERTMSG(offset < softint_bytes, "%"PRIuPTR" %u",
    502 	    offset, softint_bytes);
    503 	sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset);
    504 
    505 	/* If it's already pending there's nothing to do. */
    506 	if ((sh->sh_flags & SOFTINT_PENDING) != 0) {
    507 		return;
    508 	}
    509 
    510 	/*
    511 	 * Enqueue the handler into the LWP's pending list.
    512 	 * If the LWP is completely idle, then make it run.
    513 	 */
    514 	s = splhigh();
    515 	if ((sh->sh_flags & SOFTINT_PENDING) == 0) {
    516 		si = sh->sh_isr;
    517 		sh->sh_flags |= SOFTINT_PENDING;
    518 		SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q);
    519 		if (si->si_active == 0) {
    520 			si->si_active = 1;
    521 			softint_trigger(si->si_machdep);
    522 		}
    523 	}
    524 	splx(s);
    525 }
    526 
    527 /*
    528  * softint_schedule_cpu:
    529  *
    530  *	Trigger a software interrupt on a target CPU.  This invokes
    531  *	softint_schedule() for the local CPU or send an IPI to invoke
    532  *	this routine on the remote CPU.  Preemption must be disabled.
    533  */
    534 void
    535 softint_schedule_cpu(void *arg, struct cpu_info *ci)
    536 {
    537 	KASSERT(kpreempt_disabled());
    538 
    539 	if (curcpu() != ci) {
    540 		const softcpu_t *sc = ci->ci_data.cpu_softcpu;
    541 		const uintptr_t offset = (uintptr_t)arg;
    542 		const softhand_t *sh;
    543 
    544 		SDT_PROBE2(sdt, kernel, softint, schedule,  arg, ci);
    545 		sh = (const softhand_t *)((const uint8_t *)sc + offset);
    546 		KASSERT((sh->sh_flags & SOFTINT_RCPU) != 0);
    547 		ipi_trigger(sh->sh_ipi_id, ci);
    548 		return;
    549 	}
    550 
    551 	/* Just a local CPU. */
    552 	softint_schedule(arg);
    553 }
    554 
    555 /*
    556  * softint_execute:
    557  *
    558  *	Invoke handlers for the specified soft interrupt.
    559  *	Must be entered at splhigh.  Will drop the priority
    560  *	to the level specified, but returns back at splhigh.
    561  */
    562 static inline void
    563 softint_execute(lwp_t *l, int s)
    564 {
    565 	softint_t *si = l->l_private;
    566 	softhand_t *sh;
    567 
    568 	KASSERT(si->si_lwp == curlwp);
    569 	KASSERT(si->si_cpu == curcpu());
    570 	KASSERT(si->si_lwp->l_wchan == NULL);
    571 	KASSERT(si->si_active);
    572 
    573 	/*
    574 	 * Note: due to priority inheritance we may have interrupted a
    575 	 * higher priority LWP.  Since the soft interrupt must be quick
    576 	 * and is non-preemptable, we don't bother yielding.
    577 	 */
    578 
    579 	while (!SIMPLEQ_EMPTY(&si->si_q)) {
    580 		/*
    581 		 * Pick the longest waiting handler to run.  We block
    582 		 * interrupts but do not lock in order to do this, as
    583 		 * we are protecting against the local CPU only.
    584 		 */
    585 		sh = SIMPLEQ_FIRST(&si->si_q);
    586 		SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q);
    587 		KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0);
    588 		sh->sh_flags ^= SOFTINT_PENDING;
    589 		splx(s);
    590 
    591 		/* Run the handler. */
    592 		SDT_PROBE4(sdt, kernel, softint, entry,
    593 		    ((const char *)sh -
    594 			(const char *)curcpu()->ci_data.cpu_softcpu),
    595 		    sh->sh_func, sh->sh_arg, sh->sh_flags);
    596 		if (__predict_true((sh->sh_flags & SOFTINT_MPSAFE) != 0)) {
    597 			(*sh->sh_func)(sh->sh_arg);
    598 		} else {
    599 			KERNEL_LOCK(1, l);
    600 			(*sh->sh_func)(sh->sh_arg);
    601 			KERNEL_UNLOCK_ONE(l);
    602 		}
    603 		SDT_PROBE4(sdt, kernel, softint, return,
    604 		    ((const char *)sh -
    605 			(const char *)curcpu()->ci_data.cpu_softcpu),
    606 		    sh->sh_func, sh->sh_arg, sh->sh_flags);
    607 
    608 		/* Diagnostic: check that spin-locks have not leaked. */
    609 		KASSERTMSG(curcpu()->ci_mtx_count == 0,
    610 		    "%s: ci_mtx_count (%d) != 0, sh_func %p\n",
    611 		    __func__, curcpu()->ci_mtx_count, sh->sh_func);
    612 		/* Diagnostic: check that psrefs have not leaked. */
    613 		KASSERTMSG(l->l_psrefs == 0, "%s: l_psrefs=%d, sh_func=%p\n",
    614 		    __func__, l->l_psrefs, sh->sh_func);
    615 		/* Diagnostic: check that biglocks have not leaked. */
    616 		KASSERTMSG(l->l_blcnt == 0,
    617 		    "%s: sh_func=%p leaked %d biglocks",
    618 		    __func__, sh->sh_func, curlwp->l_blcnt);
    619 
    620 		(void)splhigh();
    621 	}
    622 
    623 	PSREF_DEBUG_BARRIER();
    624 
    625 	CPU_COUNT(CPU_COUNT_NSOFT, 1);
    626 
    627 	KASSERT(si->si_cpu == curcpu());
    628 	KASSERT(si->si_lwp->l_wchan == NULL);
    629 	KASSERT(si->si_active);
    630 	si->si_evcnt.ev_count++;
    631 	si->si_active = 0;
    632 }
    633 
    634 /*
    635  * softint_block:
    636  *
    637  *	Update statistics when the soft interrupt blocks.
    638  */
    639 void
    640 softint_block(lwp_t *l)
    641 {
    642 	softint_t *si = l->l_private;
    643 
    644 	KASSERT((l->l_pflag & LP_INTR) != 0);
    645 	si->si_evcnt_block.ev_count++;
    646 }
    647 
    648 #ifndef __HAVE_FAST_SOFTINTS
    649 
    650 #ifdef __HAVE_PREEMPTION
    651 #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS
    652 #endif
    653 
    654 /*
    655  * softint_init_md:
    656  *
    657  *	Slow path: perform machine-dependent initialization.
    658  */
    659 void
    660 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
    661 {
    662 	struct proc *p;
    663 	softint_t *si;
    664 
    665 	*machdep = (1 << level);
    666 	si = l->l_private;
    667 	p = l->l_proc;
    668 
    669 	mutex_enter(p->p_lock);
    670 	lwp_lock(l);
    671 	/* Cheat and make the KASSERT in softint_thread() happy. */
    672 	si->si_active = 1;
    673 	setrunnable(l);
    674 	/* LWP now unlocked */
    675 	mutex_exit(p->p_lock);
    676 }
    677 
    678 /*
    679  * softint_trigger:
    680  *
    681  *	Slow path: cause a soft interrupt handler to begin executing.
    682  *	Called at IPL_HIGH.
    683  */
    684 void
    685 softint_trigger(uintptr_t machdep)
    686 {
    687 	struct cpu_info *ci;
    688 	lwp_t *l;
    689 
    690 	ci = curcpu();
    691 	ci->ci_data.cpu_softints |= machdep;
    692 	l = ci->ci_onproc;
    693 
    694 	/*
    695 	 * Arrange for mi_switch() to be called.  If called from interrupt
    696 	 * mode, we don't know if curlwp is executing in kernel or user, so
    697 	 * post an AST and have it take a trip through userret().  If not in
    698 	 * interrupt mode, curlwp is running in kernel and will notice the
    699 	 * resched soon enough; avoid the AST.
    700 	 */
    701 	if (l == ci->ci_data.cpu_idlelwp) {
    702 		atomic_or_uint(&ci->ci_want_resched,
    703 		    RESCHED_IDLE | RESCHED_UPREEMPT);
    704 	} else {
    705 		atomic_or_uint(&ci->ci_want_resched, RESCHED_UPREEMPT);
    706 		if (cpu_intr_p()) {
    707 			cpu_signotify(l);
    708 		}
    709 	}
    710 }
    711 
    712 /*
    713  * softint_thread:
    714  *
    715  *	Slow path: MI software interrupt dispatch.
    716  */
    717 void
    718 softint_thread(void *cookie)
    719 {
    720 	softint_t *si;
    721 	lwp_t *l;
    722 	int s;
    723 
    724 	l = curlwp;
    725 	si = l->l_private;
    726 
    727 	for (;;) {
    728 		/* Clear pending status and run it. */
    729 		s = splhigh();
    730 		l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep;
    731 		softint_execute(l, s);
    732 		splx(s);
    733 
    734 		/* Interrupts allowed to run again before switching. */
    735 		lwp_lock(l);
    736 		l->l_stat = LSIDL;
    737 		spc_lock(l->l_cpu);
    738 		mi_switch(l);
    739 	}
    740 }
    741 
    742 /*
    743  * softint_picklwp:
    744  *
    745  *	Slow path: called from mi_switch() to pick the highest priority
    746  *	soft interrupt LWP that needs to run.
    747  */
    748 lwp_t *
    749 softint_picklwp(void)
    750 {
    751 	struct cpu_info *ci;
    752 	u_int mask;
    753 	softint_t *si;
    754 	lwp_t *l;
    755 
    756 	ci = curcpu();
    757 	si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
    758 	mask = ci->ci_data.cpu_softints;
    759 
    760 	if ((mask & (1 << SOFTINT_SERIAL)) != 0) {
    761 		l = si[SOFTINT_SERIAL].si_lwp;
    762 	} else if ((mask & (1 << SOFTINT_NET)) != 0) {
    763 		l = si[SOFTINT_NET].si_lwp;
    764 	} else if ((mask & (1 << SOFTINT_BIO)) != 0) {
    765 		l = si[SOFTINT_BIO].si_lwp;
    766 	} else if ((mask & (1 << SOFTINT_CLOCK)) != 0) {
    767 		l = si[SOFTINT_CLOCK].si_lwp;
    768 	} else {
    769 		panic("softint_picklwp");
    770 	}
    771 
    772 	return l;
    773 }
    774 
    775 #else	/*  !__HAVE_FAST_SOFTINTS */
    776 
    777 /*
    778  * softint_thread:
    779  *
    780  *	Fast path: the LWP is switched to without restoring any state,
    781  *	so we should not arrive here - there is a direct handoff between
    782  *	the interrupt stub and softint_dispatch().
    783  */
    784 void
    785 softint_thread(void *cookie)
    786 {
    787 
    788 	panic("softint_thread");
    789 }
    790 
    791 /*
    792  * softint_dispatch:
    793  *
    794  *	Fast path: entry point from machine-dependent code.
    795  */
    796 void
    797 softint_dispatch(lwp_t *pinned, int s)
    798 {
    799 	struct bintime now;
    800 	u_int timing;
    801 	lwp_t *l;
    802 
    803 #ifdef DIAGNOSTIC
    804 	if ((pinned->l_pflag & LP_RUNNING) == 0 || curlwp->l_stat != LSIDL) {
    805 		struct lwp *onproc = curcpu()->ci_onproc;
    806 		int s2 = splhigh();
    807 		printf("curcpu=%d, spl=%d curspl=%d\n"
    808 			"onproc=%p => l_stat=%d l_flag=%08x l_cpu=%d\n"
    809 			"curlwp=%p => l_stat=%d l_flag=%08x l_cpu=%d\n"
    810 			"pinned=%p => l_stat=%d l_flag=%08x l_cpu=%d\n",
    811 			cpu_index(curcpu()), s, s2, onproc, onproc->l_stat,
    812 			onproc->l_flag, cpu_index(onproc->l_cpu), curlwp,
    813 			curlwp->l_stat, curlwp->l_flag,
    814 			cpu_index(curlwp->l_cpu), pinned, pinned->l_stat,
    815 			pinned->l_flag, cpu_index(pinned->l_cpu));
    816 		splx(s2);
    817 		panic("softint screwup");
    818 	}
    819 #endif
    820 
    821 	/*
    822 	 * Note the interrupted LWP, and mark the current LWP as running
    823 	 * before proceeding.  Although this must as a rule be done with
    824 	 * the LWP locked, at this point no external agents will want to
    825 	 * modify the interrupt LWP's state.
    826 	 */
    827 	timing = softint_timing;
    828 	l = curlwp;
    829 	l->l_switchto = pinned;
    830 	l->l_stat = LSONPROC;
    831 
    832 	/*
    833 	 * Dispatch the interrupt.  If softints are being timed, charge
    834 	 * for it.
    835 	 */
    836 	if (timing) {
    837 		binuptime(&l->l_stime);
    838 		membar_producer();	/* for calcru */
    839 		l->l_pflag |= LP_TIMEINTR;
    840 	}
    841 	l->l_pflag |= LP_RUNNING;
    842 	softint_execute(l, s);
    843 	if (timing) {
    844 		binuptime(&now);
    845 		updatertime(l, &now);
    846 		l->l_pflag &= ~LP_TIMEINTR;
    847 	}
    848 
    849 	/*
    850 	 * If we blocked while handling the interrupt, the pinned LWP is
    851 	 * gone and we are now running as a kthread, so find another LWP to
    852 	 * run.  softint_dispatch() won't be reentered until the priority is
    853 	 * finally dropped to IPL_NONE on entry to the next LWP on this CPU.
    854 	 */
    855 	l->l_stat = LSIDL;
    856 	if (l->l_switchto == NULL) {
    857 		lwp_lock(l);
    858 		spc_lock(l->l_cpu);
    859 		mi_switch(l);
    860 		/* NOTREACHED */
    861 	}
    862 	l->l_switchto = NULL;
    863 	l->l_pflag &= ~LP_RUNNING;
    864 }
    865 
    866 #endif	/* !__HAVE_FAST_SOFTINTS */
    867