kern_softint.c revision 1.30 1 1.30 rmind /* $NetBSD: kern_softint.c,v 1.30 2010/01/08 12:10:46 rmind Exp $ */
2 1.2 ad
3 1.2 ad /*-
4 1.10 ad * Copyright (c) 2007, 2008 The NetBSD Foundation, Inc.
5 1.2 ad * All rights reserved.
6 1.2 ad *
7 1.2 ad * This code is derived from software contributed to The NetBSD Foundation
8 1.2 ad * by Andrew Doran.
9 1.2 ad *
10 1.2 ad * Redistribution and use in source and binary forms, with or without
11 1.2 ad * modification, are permitted provided that the following conditions
12 1.2 ad * are met:
13 1.2 ad * 1. Redistributions of source code must retain the above copyright
14 1.2 ad * notice, this list of conditions and the following disclaimer.
15 1.2 ad * 2. Redistributions in binary form must reproduce the above copyright
16 1.2 ad * notice, this list of conditions and the following disclaimer in the
17 1.2 ad * documentation and/or other materials provided with the distribution.
18 1.2 ad *
19 1.2 ad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.2 ad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.2 ad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.2 ad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.2 ad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.2 ad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.2 ad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.2 ad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.2 ad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.2 ad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.2 ad * POSSIBILITY OF SUCH DAMAGE.
30 1.2 ad */
31 1.2 ad
32 1.2 ad /*
33 1.5 ad * Generic software interrupt framework.
34 1.5 ad *
35 1.5 ad * Overview
36 1.5 ad *
37 1.5 ad * The soft interrupt framework provides a mechanism to schedule a
38 1.5 ad * low priority callback that runs with thread context. It allows
39 1.5 ad * for dynamic registration of software interrupts, and for fair
40 1.5 ad * queueing and prioritization of those interrupts. The callbacks
41 1.5 ad * can be scheduled to run from nearly any point in the kernel: by
42 1.5 ad * code running with thread context, by code running from a
43 1.5 ad * hardware interrupt handler, and at any interrupt priority
44 1.5 ad * level.
45 1.5 ad *
46 1.5 ad * Priority levels
47 1.5 ad *
48 1.5 ad * Since soft interrupt dispatch can be tied to the underlying
49 1.5 ad * architecture's interrupt dispatch code, it can be limited
50 1.5 ad * both by the capabilities of the hardware and the capabilities
51 1.5 ad * of the interrupt dispatch code itself. The number of priority
52 1.5 ad * levels is restricted to four. In order of priority (lowest to
53 1.5 ad * highest) the levels are: clock, bio, net, serial.
54 1.5 ad *
55 1.5 ad * The names are symbolic and in isolation do not have any direct
56 1.5 ad * connection with a particular kind of device activity: they are
57 1.5 ad * only meant as a guide.
58 1.5 ad *
59 1.5 ad * The four priority levels map directly to scheduler priority
60 1.5 ad * levels, and where the architecture implements 'fast' software
61 1.5 ad * interrupts, they also map onto interrupt priorities. The
62 1.5 ad * interrupt priorities are intended to be hidden from machine
63 1.5 ad * independent code, which should use thread-safe mechanisms to
64 1.5 ad * synchronize with software interrupts (for example: mutexes).
65 1.5 ad *
66 1.5 ad * Capabilities
67 1.5 ad *
68 1.5 ad * Software interrupts run with limited machine context. In
69 1.5 ad * particular, they do not posess any address space context. They
70 1.5 ad * should not try to operate on user space addresses, or to use
71 1.5 ad * virtual memory facilities other than those noted as interrupt
72 1.5 ad * safe.
73 1.5 ad *
74 1.5 ad * Unlike hardware interrupts, software interrupts do have thread
75 1.5 ad * context. They may block on synchronization objects, sleep, and
76 1.5 ad * resume execution at a later time.
77 1.5 ad *
78 1.5 ad * Since software interrupts are a limited resource and run with
79 1.5 ad * higher priority than most other LWPs in the system, all
80 1.5 ad * block-and-resume activity by a software interrupt must be kept
81 1.5 ad * short to allow futher processing at that level to continue. By
82 1.5 ad * extension, code running with process context must take care to
83 1.5 ad * ensure that any lock that may be taken from a software interrupt
84 1.5 ad * can not be held for more than a short period of time.
85 1.5 ad *
86 1.5 ad * The kernel does not allow software interrupts to use facilities
87 1.5 ad * or perform actions that may block for a significant amount of
88 1.5 ad * time. This means that it's not valid for a software interrupt
89 1.10 ad * to sleep on condition variables or wait for resources to become
90 1.10 ad * available (for example, memory).
91 1.5 ad *
92 1.5 ad * Per-CPU operation
93 1.5 ad *
94 1.5 ad * If a soft interrupt is triggered on a CPU, it can only be
95 1.5 ad * dispatched on the same CPU. Each LWP dedicated to handling a
96 1.5 ad * soft interrupt is bound to its home CPU, so if the LWP blocks
97 1.5 ad * and needs to run again, it can only run there. Nearly all data
98 1.5 ad * structures used to manage software interrupts are per-CPU.
99 1.5 ad *
100 1.5 ad * The per-CPU requirement is intended to reduce "ping-pong" of
101 1.5 ad * cache lines between CPUs: lines occupied by data structures
102 1.5 ad * used to manage the soft interrupts, and lines occupied by data
103 1.5 ad * items being passed down to the soft interrupt. As a positive
104 1.5 ad * side effect, this also means that the soft interrupt dispatch
105 1.5 ad * code does not need to to use spinlocks to synchronize.
106 1.5 ad *
107 1.5 ad * Generic implementation
108 1.5 ad *
109 1.5 ad * A generic, low performance implementation is provided that
110 1.5 ad * works across all architectures, with no machine-dependent
111 1.5 ad * modifications needed. This implementation uses the scheduler,
112 1.5 ad * and so has a number of restrictions:
113 1.5 ad *
114 1.5 ad * 1) The software interrupts are not currently preemptive, so
115 1.5 ad * must wait for the currently executing LWP to yield the CPU.
116 1.5 ad * This can introduce latency.
117 1.5 ad *
118 1.5 ad * 2) An expensive context switch is required for a software
119 1.5 ad * interrupt to be handled.
120 1.5 ad *
121 1.5 ad * 'Fast' software interrupts
122 1.5 ad *
123 1.5 ad * If an architectures defines __HAVE_FAST_SOFTINTS, it implements
124 1.5 ad * the fast mechanism. Threads running either in the kernel or in
125 1.5 ad * userspace will be interrupted, but will not be preempted. When
126 1.5 ad * the soft interrupt completes execution, the interrupted LWP
127 1.5 ad * is resumed. Interrupt dispatch code must provide the minimum
128 1.5 ad * level of context necessary for the soft interrupt to block and
129 1.5 ad * be resumed at a later time. The machine-dependent dispatch
130 1.5 ad * path looks something like the following:
131 1.5 ad *
132 1.5 ad * softintr()
133 1.5 ad * {
134 1.5 ad * go to IPL_HIGH if necessary for switch;
135 1.5 ad * save any necessary registers in a format that can be
136 1.5 ad * restored by cpu_switchto if the softint blocks;
137 1.5 ad * arrange for cpu_switchto() to restore into the
138 1.5 ad * trampoline function;
139 1.5 ad * identify LWP to handle this interrupt;
140 1.5 ad * switch to the LWP's stack;
141 1.5 ad * switch register stacks, if necessary;
142 1.5 ad * assign new value of curlwp;
143 1.5 ad * call MI softint_dispatch, passing old curlwp and IPL
144 1.5 ad * to execute interrupt at;
145 1.5 ad * switch back to old stack;
146 1.5 ad * switch back to old register stack, if necessary;
147 1.5 ad * restore curlwp;
148 1.5 ad * return to interrupted LWP;
149 1.5 ad * }
150 1.5 ad *
151 1.5 ad * If the soft interrupt blocks, a trampoline function is returned
152 1.5 ad * to in the context of the interrupted LWP, as arranged for by
153 1.5 ad * softint():
154 1.5 ad *
155 1.5 ad * softint_ret()
156 1.5 ad * {
157 1.5 ad * unlock soft interrupt LWP;
158 1.5 ad * resume interrupt processing, likely returning to
159 1.5 ad * interrupted LWP or dispatching another, different
160 1.5 ad * interrupt;
161 1.5 ad * }
162 1.5 ad *
163 1.5 ad * Once the soft interrupt has fired (and even if it has blocked),
164 1.5 ad * no further soft interrupts at that level will be triggered by
165 1.5 ad * MI code until the soft interrupt handler has ceased execution.
166 1.5 ad * If a soft interrupt handler blocks and is resumed, it resumes
167 1.5 ad * execution as a normal LWP (kthread) and gains VM context. Only
168 1.5 ad * when it has completed and is ready to fire again will it
169 1.5 ad * interrupt other threads.
170 1.5 ad *
171 1.5 ad * Future directions
172 1.5 ad *
173 1.5 ad * Provide a cheap way to direct software interrupts to remote
174 1.5 ad * CPUs. Provide a way to enqueue work items into the handler
175 1.5 ad * record, removing additional spl calls (see subr_workqueue.c).
176 1.2 ad */
177 1.2 ad
178 1.2 ad #include <sys/cdefs.h>
179 1.30 rmind __KERNEL_RCSID(0, "$NetBSD: kern_softint.c,v 1.30 2010/01/08 12:10:46 rmind Exp $");
180 1.2 ad
181 1.2 ad #include <sys/param.h>
182 1.5 ad #include <sys/malloc.h>
183 1.5 ad #include <sys/proc.h>
184 1.2 ad #include <sys/intr.h>
185 1.5 ad #include <sys/mutex.h>
186 1.5 ad #include <sys/kthread.h>
187 1.5 ad #include <sys/evcnt.h>
188 1.5 ad #include <sys/cpu.h>
189 1.24 ad #include <sys/xcall.h>
190 1.5 ad
191 1.5 ad #include <net/netisr.h>
192 1.5 ad
193 1.5 ad #include <uvm/uvm_extern.h>
194 1.5 ad
195 1.5 ad /* This could overlap with signal info in struct lwp. */
196 1.5 ad typedef struct softint {
197 1.5 ad SIMPLEQ_HEAD(, softhand) si_q;
198 1.5 ad struct lwp *si_lwp;
199 1.5 ad struct cpu_info *si_cpu;
200 1.5 ad uintptr_t si_machdep;
201 1.5 ad struct evcnt si_evcnt;
202 1.5 ad struct evcnt si_evcnt_block;
203 1.5 ad int si_active;
204 1.5 ad char si_name[8];
205 1.5 ad char si_name_block[8+6];
206 1.5 ad } softint_t;
207 1.5 ad
208 1.5 ad typedef struct softhand {
209 1.5 ad SIMPLEQ_ENTRY(softhand) sh_q;
210 1.5 ad void (*sh_func)(void *);
211 1.5 ad void *sh_arg;
212 1.5 ad softint_t *sh_isr;
213 1.28 bouyer u_int sh_flags;
214 1.5 ad } softhand_t;
215 1.5 ad
216 1.5 ad typedef struct softcpu {
217 1.5 ad struct cpu_info *sc_cpu;
218 1.5 ad softint_t sc_int[SOFTINT_COUNT];
219 1.5 ad softhand_t sc_hand[1];
220 1.5 ad } softcpu_t;
221 1.5 ad
222 1.5 ad static void softint_thread(void *);
223 1.5 ad
224 1.5 ad u_int softint_bytes = 8192;
225 1.5 ad u_int softint_timing;
226 1.5 ad static u_int softint_max;
227 1.5 ad static kmutex_t softint_lock;
228 1.23 pooka static void *softint_netisrs[NETISR_MAX];
229 1.2 ad
230 1.5 ad /*
231 1.5 ad * softint_init_isr:
232 1.5 ad *
233 1.5 ad * Initialize a single interrupt level for a single CPU.
234 1.5 ad */
235 1.5 ad static void
236 1.5 ad softint_init_isr(softcpu_t *sc, const char *desc, pri_t pri, u_int level)
237 1.5 ad {
238 1.5 ad struct cpu_info *ci;
239 1.5 ad softint_t *si;
240 1.5 ad int error;
241 1.5 ad
242 1.5 ad si = &sc->sc_int[level];
243 1.5 ad ci = sc->sc_cpu;
244 1.5 ad si->si_cpu = ci;
245 1.5 ad
246 1.5 ad SIMPLEQ_INIT(&si->si_q);
247 1.5 ad
248 1.5 ad error = kthread_create(pri, KTHREAD_MPSAFE | KTHREAD_INTR |
249 1.5 ad KTHREAD_IDLE, ci, softint_thread, si, &si->si_lwp,
250 1.12 martin "soft%s/%u", desc, ci->ci_index);
251 1.5 ad if (error != 0)
252 1.5 ad panic("softint_init_isr: error %d", error);
253 1.5 ad
254 1.12 martin snprintf(si->si_name, sizeof(si->si_name), "%s/%u", desc,
255 1.12 martin ci->ci_index);
256 1.20 ad evcnt_attach_dynamic(&si->si_evcnt, EVCNT_TYPE_MISC, NULL,
257 1.5 ad "softint", si->si_name);
258 1.12 martin snprintf(si->si_name_block, sizeof(si->si_name_block), "%s block/%u",
259 1.12 martin desc, ci->ci_index);
260 1.20 ad evcnt_attach_dynamic(&si->si_evcnt_block, EVCNT_TYPE_MISC, NULL,
261 1.5 ad "softint", si->si_name_block);
262 1.3 ad
263 1.5 ad si->si_lwp->l_private = si;
264 1.5 ad softint_init_md(si->si_lwp, level, &si->si_machdep);
265 1.5 ad }
266 1.2 ad /*
267 1.2 ad * softint_init:
268 1.2 ad *
269 1.2 ad * Initialize per-CPU data structures. Called from mi_cpu_attach().
270 1.2 ad */
271 1.2 ad void
272 1.2 ad softint_init(struct cpu_info *ci)
273 1.2 ad {
274 1.5 ad static struct cpu_info *first;
275 1.5 ad softcpu_t *sc, *scfirst;
276 1.5 ad softhand_t *sh, *shmax;
277 1.5 ad
278 1.5 ad if (first == NULL) {
279 1.5 ad /* Boot CPU. */
280 1.5 ad first = ci;
281 1.5 ad mutex_init(&softint_lock, MUTEX_DEFAULT, IPL_NONE);
282 1.5 ad softint_bytes = round_page(softint_bytes);
283 1.5 ad softint_max = (softint_bytes - sizeof(softcpu_t)) /
284 1.5 ad sizeof(softhand_t);
285 1.5 ad }
286 1.2 ad
287 1.5 ad sc = (softcpu_t *)uvm_km_alloc(kernel_map, softint_bytes, 0,
288 1.5 ad UVM_KMF_WIRED | UVM_KMF_ZERO);
289 1.5 ad if (sc == NULL)
290 1.5 ad panic("softint_init_cpu: cannot allocate memory");
291 1.5 ad
292 1.5 ad ci->ci_data.cpu_softcpu = sc;
293 1.5 ad ci->ci_data.cpu_softints = 0;
294 1.5 ad sc->sc_cpu = ci;
295 1.5 ad
296 1.5 ad softint_init_isr(sc, "net", PRI_SOFTNET, SOFTINT_NET);
297 1.5 ad softint_init_isr(sc, "bio", PRI_SOFTBIO, SOFTINT_BIO);
298 1.5 ad softint_init_isr(sc, "clk", PRI_SOFTCLOCK, SOFTINT_CLOCK);
299 1.5 ad softint_init_isr(sc, "ser", PRI_SOFTSERIAL, SOFTINT_SERIAL);
300 1.5 ad
301 1.5 ad if (first != ci) {
302 1.5 ad mutex_enter(&softint_lock);
303 1.5 ad scfirst = first->ci_data.cpu_softcpu;
304 1.5 ad sh = sc->sc_hand;
305 1.5 ad memcpy(sh, scfirst->sc_hand, sizeof(*sh) * softint_max);
306 1.5 ad /* Update pointers for this CPU. */
307 1.5 ad for (shmax = sh + softint_max; sh < shmax; sh++) {
308 1.5 ad if (sh->sh_func == NULL)
309 1.5 ad continue;
310 1.5 ad sh->sh_isr =
311 1.5 ad &sc->sc_int[sh->sh_flags & SOFTINT_LVLMASK];
312 1.5 ad }
313 1.5 ad mutex_exit(&softint_lock);
314 1.5 ad } else {
315 1.5 ad /*
316 1.5 ad * Establish handlers for legacy net interrupts.
317 1.5 ad * XXX Needs to go away.
318 1.5 ad */
319 1.5 ad #define DONETISR(n, f) \
320 1.16 ad softint_netisrs[(n)] = softint_establish(SOFTINT_NET|SOFTINT_MPSAFE,\
321 1.16 ad (void (*)(void *))(f), NULL)
322 1.5 ad #include <net/netisr_dispatch.h>
323 1.5 ad }
324 1.2 ad }
325 1.2 ad
326 1.2 ad /*
327 1.2 ad * softint_establish:
328 1.2 ad *
329 1.2 ad * Register a software interrupt handler.
330 1.2 ad */
331 1.2 ad void *
332 1.2 ad softint_establish(u_int flags, void (*func)(void *), void *arg)
333 1.2 ad {
334 1.5 ad CPU_INFO_ITERATOR cii;
335 1.5 ad struct cpu_info *ci;
336 1.5 ad softcpu_t *sc;
337 1.5 ad softhand_t *sh;
338 1.5 ad u_int level, index;
339 1.2 ad
340 1.2 ad level = (flags & SOFTINT_LVLMASK);
341 1.2 ad KASSERT(level < SOFTINT_COUNT);
342 1.24 ad KASSERT((flags & SOFTINT_IMPMASK) == 0);
343 1.2 ad
344 1.5 ad mutex_enter(&softint_lock);
345 1.5 ad
346 1.5 ad /* Find a free slot. */
347 1.5 ad sc = curcpu()->ci_data.cpu_softcpu;
348 1.5 ad for (index = 1; index < softint_max; index++)
349 1.5 ad if (sc->sc_hand[index].sh_func == NULL)
350 1.5 ad break;
351 1.5 ad if (index == softint_max) {
352 1.5 ad mutex_exit(&softint_lock);
353 1.5 ad printf("WARNING: softint_establish: table full, "
354 1.5 ad "increase softint_bytes\n");
355 1.5 ad return NULL;
356 1.5 ad }
357 1.5 ad
358 1.5 ad /* Set up the handler on each CPU. */
359 1.8 ad if (ncpu < 2) {
360 1.7 ad /* XXX hack for machines with no CPU_INFO_FOREACH() early on */
361 1.7 ad sc = curcpu()->ci_data.cpu_softcpu;
362 1.7 ad sh = &sc->sc_hand[index];
363 1.7 ad sh->sh_isr = &sc->sc_int[level];
364 1.7 ad sh->sh_func = func;
365 1.7 ad sh->sh_arg = arg;
366 1.7 ad sh->sh_flags = flags;
367 1.7 ad } else for (CPU_INFO_FOREACH(cii, ci)) {
368 1.5 ad sc = ci->ci_data.cpu_softcpu;
369 1.5 ad sh = &sc->sc_hand[index];
370 1.5 ad sh->sh_isr = &sc->sc_int[level];
371 1.5 ad sh->sh_func = func;
372 1.5 ad sh->sh_arg = arg;
373 1.5 ad sh->sh_flags = flags;
374 1.2 ad }
375 1.2 ad
376 1.5 ad mutex_exit(&softint_lock);
377 1.5 ad
378 1.5 ad return (void *)((uint8_t *)&sc->sc_hand[index] - (uint8_t *)sc);
379 1.2 ad }
380 1.2 ad
381 1.2 ad /*
382 1.2 ad * softint_disestablish:
383 1.2 ad *
384 1.24 ad * Unregister a software interrupt handler. The soft interrupt could
385 1.24 ad * still be active at this point, but the caller commits not to try
386 1.24 ad * and trigger it again once this call is made. The caller must not
387 1.24 ad * hold any locks that could be taken from soft interrupt context,
388 1.24 ad * because we will wait for the softint to complete if it's still
389 1.24 ad * running.
390 1.2 ad */
391 1.2 ad void
392 1.2 ad softint_disestablish(void *arg)
393 1.2 ad {
394 1.5 ad CPU_INFO_ITERATOR cii;
395 1.5 ad struct cpu_info *ci;
396 1.5 ad softcpu_t *sc;
397 1.5 ad softhand_t *sh;
398 1.5 ad uintptr_t offset;
399 1.24 ad uint64_t where;
400 1.24 ad u_int flags;
401 1.5 ad
402 1.5 ad offset = (uintptr_t)arg;
403 1.5 ad KASSERT(offset != 0 && offset < softint_bytes);
404 1.5 ad
405 1.24 ad /*
406 1.24 ad * Run a cross call so we see up to date values of sh_flags from
407 1.24 ad * all CPUs. Once softint_disestablish() is called, the caller
408 1.24 ad * commits to not trigger the interrupt and set SOFTINT_ACTIVE on
409 1.24 ad * it again. So, we are only looking for handler records with
410 1.26 dyoung * SOFTINT_ACTIVE already set.
411 1.24 ad */
412 1.24 ad where = xc_broadcast(0, (xcfunc_t)nullop, NULL, NULL);
413 1.24 ad xc_wait(where);
414 1.24 ad
415 1.24 ad for (;;) {
416 1.24 ad /* Collect flag values from each CPU. */
417 1.24 ad flags = 0;
418 1.24 ad for (CPU_INFO_FOREACH(cii, ci)) {
419 1.24 ad sc = ci->ci_data.cpu_softcpu;
420 1.24 ad sh = (softhand_t *)((uint8_t *)sc + offset);
421 1.24 ad KASSERT(sh->sh_func != NULL);
422 1.24 ad flags |= sh->sh_flags;
423 1.24 ad }
424 1.24 ad /* Inactive on all CPUs? */
425 1.24 ad if ((flags & SOFTINT_ACTIVE) == 0) {
426 1.24 ad break;
427 1.24 ad }
428 1.24 ad /* Oops, still active. Wait for it to clear. */
429 1.25 ad (void)kpause("softdis", false, 1, NULL);
430 1.24 ad }
431 1.5 ad
432 1.5 ad /* Clear the handler on each CPU. */
433 1.24 ad mutex_enter(&softint_lock);
434 1.5 ad for (CPU_INFO_FOREACH(cii, ci)) {
435 1.5 ad sc = ci->ci_data.cpu_softcpu;
436 1.5 ad sh = (softhand_t *)((uint8_t *)sc + offset);
437 1.5 ad KASSERT(sh->sh_func != NULL);
438 1.5 ad sh->sh_func = NULL;
439 1.5 ad }
440 1.5 ad mutex_exit(&softint_lock);
441 1.2 ad }
442 1.2 ad
443 1.2 ad /*
444 1.2 ad * softint_schedule:
445 1.2 ad *
446 1.2 ad * Trigger a software interrupt. Must be called from a hardware
447 1.2 ad * interrupt handler, or with preemption disabled (since we are
448 1.2 ad * using the value of curcpu()).
449 1.2 ad */
450 1.2 ad void
451 1.2 ad softint_schedule(void *arg)
452 1.2 ad {
453 1.5 ad softhand_t *sh;
454 1.5 ad softint_t *si;
455 1.5 ad uintptr_t offset;
456 1.5 ad int s;
457 1.5 ad
458 1.17 ad KASSERT(kpreempt_disabled());
459 1.17 ad
460 1.5 ad /* Find the handler record for this CPU. */
461 1.5 ad offset = (uintptr_t)arg;
462 1.5 ad KASSERT(offset != 0 && offset < softint_bytes);
463 1.5 ad sh = (softhand_t *)((uint8_t *)curcpu()->ci_data.cpu_softcpu + offset);
464 1.5 ad
465 1.5 ad /* If it's already pending there's nothing to do. */
466 1.24 ad if ((sh->sh_flags & SOFTINT_PENDING) != 0)
467 1.5 ad return;
468 1.5 ad
469 1.5 ad /*
470 1.5 ad * Enqueue the handler into the LWP's pending list.
471 1.5 ad * If the LWP is completely idle, then make it run.
472 1.5 ad */
473 1.5 ad s = splhigh();
474 1.24 ad if ((sh->sh_flags & SOFTINT_PENDING) == 0) {
475 1.5 ad si = sh->sh_isr;
476 1.24 ad sh->sh_flags |= SOFTINT_PENDING;
477 1.5 ad SIMPLEQ_INSERT_TAIL(&si->si_q, sh, sh_q);
478 1.5 ad if (si->si_active == 0) {
479 1.5 ad si->si_active = 1;
480 1.5 ad softint_trigger(si->si_machdep);
481 1.5 ad }
482 1.5 ad }
483 1.5 ad splx(s);
484 1.5 ad }
485 1.5 ad
486 1.5 ad /*
487 1.5 ad * softint_execute:
488 1.5 ad *
489 1.5 ad * Invoke handlers for the specified soft interrupt.
490 1.5 ad * Must be entered at splhigh. Will drop the priority
491 1.5 ad * to the level specified, but returns back at splhigh.
492 1.5 ad */
493 1.5 ad static inline void
494 1.5 ad softint_execute(softint_t *si, lwp_t *l, int s)
495 1.5 ad {
496 1.5 ad softhand_t *sh;
497 1.5 ad bool havelock;
498 1.5 ad
499 1.5 ad #ifdef __HAVE_FAST_SOFTINTS
500 1.5 ad KASSERT(si->si_lwp == curlwp);
501 1.5 ad #else
502 1.5 ad /* May be running in user context. */
503 1.5 ad #endif
504 1.5 ad KASSERT(si->si_cpu == curcpu());
505 1.5 ad KASSERT(si->si_lwp->l_wchan == NULL);
506 1.5 ad KASSERT(si->si_active);
507 1.5 ad
508 1.5 ad havelock = false;
509 1.5 ad
510 1.5 ad /*
511 1.5 ad * Note: due to priority inheritance we may have interrupted a
512 1.5 ad * higher priority LWP. Since the soft interrupt must be quick
513 1.5 ad * and is non-preemptable, we don't bother yielding.
514 1.5 ad */
515 1.5 ad
516 1.5 ad while (!SIMPLEQ_EMPTY(&si->si_q)) {
517 1.5 ad /*
518 1.5 ad * Pick the longest waiting handler to run. We block
519 1.5 ad * interrupts but do not lock in order to do this, as
520 1.5 ad * we are protecting against the local CPU only.
521 1.5 ad */
522 1.5 ad sh = SIMPLEQ_FIRST(&si->si_q);
523 1.5 ad SIMPLEQ_REMOVE_HEAD(&si->si_q, sh_q);
524 1.24 ad KASSERT((sh->sh_flags & SOFTINT_PENDING) != 0);
525 1.24 ad KASSERT((sh->sh_flags & SOFTINT_ACTIVE) == 0);
526 1.24 ad sh->sh_flags ^= (SOFTINT_PENDING | SOFTINT_ACTIVE);
527 1.5 ad splx(s);
528 1.5 ad
529 1.5 ad /* Run the handler. */
530 1.30 rmind if (sh->sh_flags & SOFTINT_MPSAFE) {
531 1.30 rmind if (havelock) {
532 1.30 rmind KERNEL_UNLOCK_ONE(l);
533 1.30 rmind havelock = false;
534 1.30 rmind }
535 1.30 rmind } else if (!havelock) {
536 1.5 ad KERNEL_LOCK(1, l);
537 1.5 ad havelock = true;
538 1.5 ad }
539 1.5 ad (*sh->sh_func)(sh->sh_arg);
540 1.5 ad
541 1.5 ad (void)splhigh();
542 1.24 ad KASSERT((sh->sh_flags & SOFTINT_ACTIVE) != 0);
543 1.24 ad sh->sh_flags ^= SOFTINT_ACTIVE;
544 1.5 ad }
545 1.2 ad
546 1.5 ad if (havelock) {
547 1.5 ad KERNEL_UNLOCK_ONE(l);
548 1.5 ad }
549 1.5 ad
550 1.5 ad /*
551 1.5 ad * Unlocked, but only for statistics.
552 1.5 ad * Should be per-CPU to prevent cache ping-pong.
553 1.5 ad */
554 1.5 ad uvmexp.softs++;
555 1.5 ad
556 1.13 ad KASSERT(si->si_cpu == curcpu());
557 1.13 ad KASSERT(si->si_lwp->l_wchan == NULL);
558 1.13 ad KASSERT(si->si_active);
559 1.5 ad si->si_evcnt.ev_count++;
560 1.5 ad si->si_active = 0;
561 1.2 ad }
562 1.2 ad
563 1.2 ad /*
564 1.2 ad * softint_block:
565 1.2 ad *
566 1.2 ad * Update statistics when the soft interrupt blocks.
567 1.2 ad */
568 1.2 ad void
569 1.2 ad softint_block(lwp_t *l)
570 1.2 ad {
571 1.5 ad softint_t *si = l->l_private;
572 1.5 ad
573 1.5 ad KASSERT((l->l_pflag & LP_INTR) != 0);
574 1.5 ad si->si_evcnt_block.ev_count++;
575 1.5 ad }
576 1.5 ad
577 1.5 ad /*
578 1.5 ad * schednetisr:
579 1.5 ad *
580 1.5 ad * Trigger a legacy network interrupt. XXX Needs to go away.
581 1.5 ad */
582 1.5 ad void
583 1.5 ad schednetisr(int isr)
584 1.5 ad {
585 1.5 ad
586 1.5 ad softint_schedule(softint_netisrs[isr]);
587 1.5 ad }
588 1.5 ad
589 1.5 ad #ifndef __HAVE_FAST_SOFTINTS
590 1.5 ad
591 1.19 ad #ifdef __HAVE_PREEMPTION
592 1.19 ad #error __HAVE_PREEMPTION requires __HAVE_FAST_SOFTINTS
593 1.17 ad #endif
594 1.17 ad
595 1.5 ad /*
596 1.5 ad * softint_init_md:
597 1.5 ad *
598 1.5 ad * Slow path: perform machine-dependent initialization.
599 1.5 ad */
600 1.5 ad void
601 1.5 ad softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep)
602 1.5 ad {
603 1.5 ad softint_t *si;
604 1.5 ad
605 1.5 ad *machdep = (1 << level);
606 1.5 ad si = l->l_private;
607 1.5 ad
608 1.5 ad lwp_lock(l);
609 1.5 ad lwp_unlock_to(l, l->l_cpu->ci_schedstate.spc_mutex);
610 1.5 ad lwp_lock(l);
611 1.5 ad /* Cheat and make the KASSERT in softint_thread() happy. */
612 1.5 ad si->si_active = 1;
613 1.5 ad l->l_stat = LSRUN;
614 1.5 ad sched_enqueue(l, false);
615 1.5 ad lwp_unlock(l);
616 1.5 ad }
617 1.5 ad
618 1.5 ad /*
619 1.5 ad * softint_trigger:
620 1.5 ad *
621 1.5 ad * Slow path: cause a soft interrupt handler to begin executing.
622 1.5 ad * Called at IPL_HIGH.
623 1.5 ad */
624 1.5 ad void
625 1.5 ad softint_trigger(uintptr_t machdep)
626 1.5 ad {
627 1.5 ad struct cpu_info *ci;
628 1.5 ad lwp_t *l;
629 1.2 ad
630 1.5 ad l = curlwp;
631 1.5 ad ci = l->l_cpu;
632 1.5 ad ci->ci_data.cpu_softints |= machdep;
633 1.5 ad if (l == ci->ci_data.cpu_idlelwp) {
634 1.5 ad cpu_need_resched(ci, 0);
635 1.5 ad } else {
636 1.5 ad /* MI equivalent of aston() */
637 1.5 ad cpu_signotify(l);
638 1.5 ad }
639 1.5 ad }
640 1.5 ad
641 1.5 ad /*
642 1.5 ad * softint_thread:
643 1.5 ad *
644 1.5 ad * Slow path: MI software interrupt dispatch.
645 1.5 ad */
646 1.5 ad void
647 1.5 ad softint_thread(void *cookie)
648 1.5 ad {
649 1.5 ad softint_t *si;
650 1.5 ad lwp_t *l;
651 1.5 ad int s;
652 1.5 ad
653 1.5 ad l = curlwp;
654 1.5 ad si = l->l_private;
655 1.5 ad
656 1.5 ad for (;;) {
657 1.5 ad /*
658 1.5 ad * Clear pending status and run it. We must drop the
659 1.5 ad * spl before mi_switch(), since IPL_HIGH may be higher
660 1.5 ad * than IPL_SCHED (and it is not safe to switch at a
661 1.5 ad * higher level).
662 1.5 ad */
663 1.5 ad s = splhigh();
664 1.5 ad l->l_cpu->ci_data.cpu_softints &= ~si->si_machdep;
665 1.5 ad softint_execute(si, l, s);
666 1.5 ad splx(s);
667 1.5 ad
668 1.5 ad lwp_lock(l);
669 1.5 ad l->l_stat = LSIDL;
670 1.5 ad mi_switch(l);
671 1.5 ad }
672 1.2 ad }
673 1.4 ad
674 1.4 ad /*
675 1.4 ad * softint_picklwp:
676 1.4 ad *
677 1.4 ad * Slow path: called from mi_switch() to pick the highest priority
678 1.4 ad * soft interrupt LWP that needs to run.
679 1.4 ad */
680 1.4 ad lwp_t *
681 1.4 ad softint_picklwp(void)
682 1.4 ad {
683 1.5 ad struct cpu_info *ci;
684 1.5 ad u_int mask;
685 1.5 ad softint_t *si;
686 1.5 ad lwp_t *l;
687 1.5 ad
688 1.5 ad ci = curcpu();
689 1.5 ad si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
690 1.5 ad mask = ci->ci_data.cpu_softints;
691 1.5 ad
692 1.5 ad if ((mask & (1 << SOFTINT_SERIAL)) != 0) {
693 1.5 ad l = si[SOFTINT_SERIAL].si_lwp;
694 1.5 ad } else if ((mask & (1 << SOFTINT_NET)) != 0) {
695 1.5 ad l = si[SOFTINT_NET].si_lwp;
696 1.5 ad } else if ((mask & (1 << SOFTINT_BIO)) != 0) {
697 1.5 ad l = si[SOFTINT_BIO].si_lwp;
698 1.5 ad } else if ((mask & (1 << SOFTINT_CLOCK)) != 0) {
699 1.5 ad l = si[SOFTINT_CLOCK].si_lwp;
700 1.5 ad } else {
701 1.5 ad panic("softint_picklwp");
702 1.5 ad }
703 1.4 ad
704 1.5 ad return l;
705 1.4 ad }
706 1.4 ad
707 1.4 ad /*
708 1.4 ad * softint_overlay:
709 1.4 ad *
710 1.4 ad * Slow path: called from lwp_userret() to run a soft interrupt
711 1.6 ad * within the context of a user thread.
712 1.4 ad */
713 1.4 ad void
714 1.4 ad softint_overlay(void)
715 1.4 ad {
716 1.5 ad struct cpu_info *ci;
717 1.14 ad u_int softints, oflag;
718 1.5 ad softint_t *si;
719 1.6 ad pri_t obase;
720 1.5 ad lwp_t *l;
721 1.5 ad int s;
722 1.5 ad
723 1.5 ad l = curlwp;
724 1.5 ad ci = l->l_cpu;
725 1.5 ad si = ((softcpu_t *)ci->ci_data.cpu_softcpu)->sc_int;
726 1.5 ad
727 1.5 ad KASSERT((l->l_pflag & LP_INTR) == 0);
728 1.5 ad
729 1.6 ad /* Arrange to elevate priority if the LWP blocks. */
730 1.14 ad s = splhigh();
731 1.6 ad obase = l->l_kpribase;
732 1.6 ad l->l_kpribase = PRI_KERNEL_RT;
733 1.14 ad oflag = l->l_pflag;
734 1.14 ad l->l_pflag = oflag | LP_INTR | LP_BOUND;
735 1.5 ad while ((softints = ci->ci_data.cpu_softints) != 0) {
736 1.5 ad if ((softints & (1 << SOFTINT_SERIAL)) != 0) {
737 1.5 ad ci->ci_data.cpu_softints &= ~(1 << SOFTINT_SERIAL);
738 1.5 ad softint_execute(&si[SOFTINT_SERIAL], l, s);
739 1.5 ad continue;
740 1.5 ad }
741 1.5 ad if ((softints & (1 << SOFTINT_NET)) != 0) {
742 1.5 ad ci->ci_data.cpu_softints &= ~(1 << SOFTINT_NET);
743 1.5 ad softint_execute(&si[SOFTINT_NET], l, s);
744 1.5 ad continue;
745 1.5 ad }
746 1.5 ad if ((softints & (1 << SOFTINT_BIO)) != 0) {
747 1.5 ad ci->ci_data.cpu_softints &= ~(1 << SOFTINT_BIO);
748 1.5 ad softint_execute(&si[SOFTINT_BIO], l, s);
749 1.5 ad continue;
750 1.5 ad }
751 1.5 ad if ((softints & (1 << SOFTINT_CLOCK)) != 0) {
752 1.5 ad ci->ci_data.cpu_softints &= ~(1 << SOFTINT_CLOCK);
753 1.5 ad softint_execute(&si[SOFTINT_CLOCK], l, s);
754 1.5 ad continue;
755 1.5 ad }
756 1.5 ad }
757 1.15 ad l->l_pflag = oflag;
758 1.14 ad l->l_kpribase = obase;
759 1.5 ad splx(s);
760 1.4 ad }
761 1.5 ad
762 1.5 ad #else /* !__HAVE_FAST_SOFTINTS */
763 1.5 ad
764 1.5 ad /*
765 1.5 ad * softint_thread:
766 1.5 ad *
767 1.5 ad * Fast path: the LWP is switched to without restoring any state,
768 1.5 ad * so we should not arrive here - there is a direct handoff between
769 1.5 ad * the interrupt stub and softint_dispatch().
770 1.5 ad */
771 1.5 ad void
772 1.5 ad softint_thread(void *cookie)
773 1.5 ad {
774 1.5 ad
775 1.5 ad panic("softint_thread");
776 1.5 ad }
777 1.5 ad
778 1.5 ad /*
779 1.5 ad * softint_dispatch:
780 1.5 ad *
781 1.5 ad * Fast path: entry point from machine-dependent code.
782 1.5 ad */
783 1.5 ad void
784 1.5 ad softint_dispatch(lwp_t *pinned, int s)
785 1.5 ad {
786 1.9 yamt struct bintime now;
787 1.5 ad softint_t *si;
788 1.5 ad u_int timing;
789 1.5 ad lwp_t *l;
790 1.5 ad
791 1.29 yamt KASSERT((pinned->l_pflag & LP_RUNNING) != 0);
792 1.5 ad l = curlwp;
793 1.5 ad si = l->l_private;
794 1.5 ad
795 1.5 ad /*
796 1.5 ad * Note the interrupted LWP, and mark the current LWP as running
797 1.5 ad * before proceeding. Although this must as a rule be done with
798 1.5 ad * the LWP locked, at this point no external agents will want to
799 1.5 ad * modify the interrupt LWP's state.
800 1.5 ad */
801 1.22 ad timing = (softint_timing ? LP_TIMEINTR : 0);
802 1.5 ad l->l_switchto = pinned;
803 1.5 ad l->l_stat = LSONPROC;
804 1.22 ad l->l_pflag |= (LP_RUNNING | timing);
805 1.5 ad
806 1.5 ad /*
807 1.5 ad * Dispatch the interrupt. If softints are being timed, charge
808 1.5 ad * for it.
809 1.5 ad */
810 1.5 ad if (timing)
811 1.11 yamt binuptime(&l->l_stime);
812 1.5 ad softint_execute(si, l, s);
813 1.5 ad if (timing) {
814 1.11 yamt binuptime(&now);
815 1.5 ad updatertime(l, &now);
816 1.22 ad l->l_pflag &= ~LP_TIMEINTR;
817 1.5 ad }
818 1.5 ad
819 1.5 ad /*
820 1.5 ad * If we blocked while handling the interrupt, the pinned LWP is
821 1.5 ad * gone so switch to the idle LWP. It will select a new LWP to
822 1.5 ad * run.
823 1.5 ad *
824 1.5 ad * We must drop the priority level as switching at IPL_HIGH could
825 1.5 ad * deadlock the system. We have already set si->si_active = 0,
826 1.5 ad * which means another interrupt at this level can be triggered.
827 1.5 ad * That's not be a problem: we are lowering to level 's' which will
828 1.5 ad * prevent softint_dispatch() from being reentered at level 's',
829 1.5 ad * until the priority is finally dropped to IPL_NONE on entry to
830 1.21 ad * the LWP chosen by lwp_exit_switchaway().
831 1.5 ad */
832 1.5 ad l->l_stat = LSIDL;
833 1.5 ad if (l->l_switchto == NULL) {
834 1.5 ad splx(s);
835 1.5 ad pmap_deactivate(l);
836 1.5 ad lwp_exit_switchaway(l);
837 1.5 ad /* NOTREACHED */
838 1.5 ad }
839 1.5 ad l->l_switchto = NULL;
840 1.22 ad l->l_pflag &= ~LP_RUNNING;
841 1.5 ad }
842 1.5 ad
843 1.5 ad #endif /* !__HAVE_FAST_SOFTINTS */
844