linux_tasklet.c revision 1.10 1 1.10 riastrad /* $NetBSD: linux_tasklet.c,v 1.10 2021/12/27 14:57:30 riastradh Exp $ */
2 1.1 riastrad
3 1.1 riastrad /*-
4 1.8 riastrad * Copyright (c) 2018, 2020, 2021 The NetBSD Foundation, Inc.
5 1.1 riastrad * All rights reserved.
6 1.1 riastrad *
7 1.1 riastrad * This code is derived from software contributed to The NetBSD Foundation
8 1.1 riastrad * by Taylor R. Campbell.
9 1.1 riastrad *
10 1.1 riastrad * Redistribution and use in source and binary forms, with or without
11 1.1 riastrad * modification, are permitted provided that the following conditions
12 1.1 riastrad * are met:
13 1.1 riastrad * 1. Redistributions of source code must retain the above copyright
14 1.1 riastrad * notice, this list of conditions and the following disclaimer.
15 1.1 riastrad * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 riastrad * notice, this list of conditions and the following disclaimer in the
17 1.1 riastrad * documentation and/or other materials provided with the distribution.
18 1.1 riastrad *
19 1.1 riastrad * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.1 riastrad * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.1 riastrad * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.1 riastrad * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.1 riastrad * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.1 riastrad * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.1 riastrad * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.1 riastrad * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.1 riastrad * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.1 riastrad * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 riastrad * POSSIBILITY OF SUCH DAMAGE.
30 1.1 riastrad */
31 1.1 riastrad
32 1.1 riastrad #include <sys/cdefs.h>
33 1.10 riastrad __KERNEL_RCSID(0, "$NetBSD: linux_tasklet.c,v 1.10 2021/12/27 14:57:30 riastradh Exp $");
34 1.1 riastrad
35 1.9 riastrad #include <sys/param.h>
36 1.1 riastrad #include <sys/types.h>
37 1.9 riastrad
38 1.1 riastrad #include <sys/atomic.h>
39 1.1 riastrad #include <sys/cpu.h>
40 1.1 riastrad #include <sys/errno.h>
41 1.1 riastrad #include <sys/intr.h>
42 1.8 riastrad #include <sys/kmem.h>
43 1.1 riastrad #include <sys/lock.h>
44 1.1 riastrad #include <sys/percpu.h>
45 1.1 riastrad #include <sys/queue.h>
46 1.1 riastrad
47 1.1 riastrad #include <lib/libkern/libkern.h>
48 1.1 riastrad
49 1.1 riastrad #include <machine/limits.h>
50 1.1 riastrad
51 1.1 riastrad #include <linux/tasklet.h>
52 1.1 riastrad
53 1.1 riastrad #define TASKLET_SCHEDULED ((unsigned)__BIT(0))
54 1.1 riastrad #define TASKLET_RUNNING ((unsigned)__BIT(1))
55 1.1 riastrad
56 1.1 riastrad struct tasklet_queue {
57 1.8 riastrad struct percpu *tq_percpu; /* struct tasklet_cpu * */
58 1.1 riastrad void *tq_sih;
59 1.1 riastrad };
60 1.1 riastrad
61 1.1 riastrad SIMPLEQ_HEAD(tasklet_head, tasklet_struct);
62 1.1 riastrad
63 1.1 riastrad struct tasklet_cpu {
64 1.1 riastrad struct tasklet_head tc_head;
65 1.1 riastrad };
66 1.1 riastrad
67 1.1 riastrad static struct tasklet_queue tasklet_queue __read_mostly;
68 1.1 riastrad static struct tasklet_queue tasklet_hi_queue __read_mostly;
69 1.1 riastrad
70 1.1 riastrad static void tasklet_softintr(void *);
71 1.1 riastrad static int tasklet_queue_init(struct tasklet_queue *, unsigned);
72 1.1 riastrad static void tasklet_queue_fini(struct tasklet_queue *);
73 1.1 riastrad static void tasklet_queue_schedule(struct tasklet_queue *,
74 1.1 riastrad struct tasklet_struct *);
75 1.1 riastrad static void tasklet_queue_enqueue(struct tasklet_queue *,
76 1.1 riastrad struct tasklet_struct *);
77 1.1 riastrad
78 1.1 riastrad /*
79 1.1 riastrad * linux_tasklets_init()
80 1.1 riastrad *
81 1.1 riastrad * Initialize the Linux tasklets subsystem. Return 0 on success,
82 1.1 riastrad * error code on failure.
83 1.1 riastrad */
84 1.1 riastrad int
85 1.1 riastrad linux_tasklets_init(void)
86 1.1 riastrad {
87 1.1 riastrad int error;
88 1.1 riastrad
89 1.1 riastrad error = tasklet_queue_init(&tasklet_queue, SOFTINT_CLOCK);
90 1.1 riastrad if (error)
91 1.1 riastrad goto fail0;
92 1.1 riastrad error = tasklet_queue_init(&tasklet_hi_queue, SOFTINT_SERIAL);
93 1.1 riastrad if (error)
94 1.1 riastrad goto fail1;
95 1.1 riastrad
96 1.1 riastrad /* Success! */
97 1.1 riastrad return 0;
98 1.1 riastrad
99 1.1 riastrad fail2: __unused
100 1.1 riastrad tasklet_queue_fini(&tasklet_hi_queue);
101 1.1 riastrad fail1: tasklet_queue_fini(&tasklet_queue);
102 1.1 riastrad fail0: KASSERT(error);
103 1.1 riastrad return error;
104 1.1 riastrad }
105 1.1 riastrad
106 1.1 riastrad /*
107 1.1 riastrad * linux_tasklets_fini()
108 1.1 riastrad *
109 1.1 riastrad * Finalize the Linux tasklets subsystem. All use of tasklets
110 1.1 riastrad * must be done.
111 1.1 riastrad */
112 1.1 riastrad void
113 1.1 riastrad linux_tasklets_fini(void)
114 1.1 riastrad {
115 1.1 riastrad
116 1.1 riastrad tasklet_queue_fini(&tasklet_hi_queue);
117 1.1 riastrad tasklet_queue_fini(&tasklet_queue);
118 1.1 riastrad }
119 1.1 riastrad
120 1.8 riastrad static void
121 1.8 riastrad tasklet_cpu_init(void *ptr, void *cookie, struct cpu_info *ci)
122 1.8 riastrad {
123 1.8 riastrad struct tasklet_cpu **tcp = ptr, *tc;
124 1.8 riastrad
125 1.8 riastrad *tcp = tc = kmem_zalloc(sizeof(*tc), KM_SLEEP);
126 1.8 riastrad SIMPLEQ_INIT(&tc->tc_head);
127 1.8 riastrad }
128 1.8 riastrad
129 1.8 riastrad static void
130 1.8 riastrad tasklet_cpu_fini(void *ptr, void *cookie, struct cpu_info *ci)
131 1.8 riastrad {
132 1.8 riastrad struct tasklet_cpu **tcp = ptr, *tc = *tcp;
133 1.8 riastrad
134 1.8 riastrad KASSERT(SIMPLEQ_EMPTY(&tc->tc_head));
135 1.8 riastrad kmem_free(tc, sizeof(*tc));
136 1.8 riastrad *tcp = NULL; /* paranoia */
137 1.8 riastrad }
138 1.8 riastrad
139 1.1 riastrad /*
140 1.1 riastrad * tasklet_queue_init(tq, prio)
141 1.1 riastrad *
142 1.1 riastrad * Initialize the tasklet queue tq for running tasklets at softint
143 1.1 riastrad * priority prio (SOFTINT_*).
144 1.1 riastrad */
145 1.1 riastrad static int
146 1.1 riastrad tasklet_queue_init(struct tasklet_queue *tq, unsigned prio)
147 1.1 riastrad {
148 1.1 riastrad int error;
149 1.1 riastrad
150 1.1 riastrad /* Allocate per-CPU memory. percpu_alloc cannot fail. */
151 1.8 riastrad tq->tq_percpu = percpu_create(sizeof(struct tasklet_cpu),
152 1.8 riastrad tasklet_cpu_init, tasklet_cpu_fini, NULL);
153 1.1 riastrad KASSERT(tq->tq_percpu != NULL);
154 1.1 riastrad
155 1.1 riastrad /* Try to establish a softint. softint_establish may fail. */
156 1.1 riastrad tq->tq_sih = softint_establish(prio|SOFTINT_MPSAFE, &tasklet_softintr,
157 1.1 riastrad tq);
158 1.1 riastrad if (tq->tq_sih == NULL) {
159 1.1 riastrad error = ENOMEM;
160 1.1 riastrad goto fail1;
161 1.1 riastrad }
162 1.1 riastrad
163 1.1 riastrad /* Success! */
164 1.1 riastrad return 0;
165 1.1 riastrad
166 1.1 riastrad fail2: __unused
167 1.1 riastrad softint_disestablish(tq->tq_sih);
168 1.1 riastrad tq->tq_sih = NULL;
169 1.1 riastrad fail1: percpu_free(tq->tq_percpu, sizeof(struct tasklet_cpu));
170 1.1 riastrad tq->tq_percpu = NULL;
171 1.1 riastrad fail0: __unused
172 1.1 riastrad KASSERT(error);
173 1.1 riastrad return error;
174 1.1 riastrad }
175 1.1 riastrad
176 1.1 riastrad /*
177 1.1 riastrad * tasklet_queue_fini(tq)
178 1.1 riastrad *
179 1.1 riastrad * Finalize the tasklet queue tq: free all resources associated
180 1.1 riastrad * with it.
181 1.1 riastrad */
182 1.1 riastrad static void
183 1.1 riastrad tasklet_queue_fini(struct tasklet_queue *tq)
184 1.1 riastrad {
185 1.1 riastrad
186 1.1 riastrad softint_disestablish(tq->tq_sih);
187 1.1 riastrad tq->tq_sih = NULL;
188 1.1 riastrad percpu_free(tq->tq_percpu, sizeof(struct tasklet_cpu));
189 1.1 riastrad tq->tq_percpu = NULL;
190 1.1 riastrad }
191 1.1 riastrad
192 1.1 riastrad /*
193 1.1 riastrad * tasklet_softintr(cookie)
194 1.1 riastrad *
195 1.1 riastrad * Soft interrupt handler: Process queued tasklets on the tasklet
196 1.1 riastrad * queue passed in as cookie.
197 1.1 riastrad */
198 1.1 riastrad static void
199 1.1 riastrad tasklet_softintr(void *cookie)
200 1.1 riastrad {
201 1.1 riastrad struct tasklet_queue *const tq = cookie;
202 1.1 riastrad struct tasklet_head th = SIMPLEQ_HEAD_INITIALIZER(th);
203 1.8 riastrad struct tasklet_cpu **tcp, *tc;
204 1.1 riastrad int s;
205 1.1 riastrad
206 1.1 riastrad /*
207 1.1 riastrad * With all interrupts deferred, transfer the current CPU's
208 1.1 riastrad * queue of tasklets to a local variable in one swell foop.
209 1.1 riastrad *
210 1.1 riastrad * No memory barriers: CPU-local state only.
211 1.1 riastrad */
212 1.8 riastrad tcp = percpu_getref(tq->tq_percpu);
213 1.8 riastrad tc = *tcp;
214 1.1 riastrad s = splhigh();
215 1.1 riastrad SIMPLEQ_CONCAT(&th, &tc->tc_head);
216 1.1 riastrad splx(s);
217 1.1 riastrad percpu_putref(tq->tq_percpu);
218 1.1 riastrad
219 1.1 riastrad /* Go through the queue of tasklets we grabbed. */
220 1.1 riastrad while (!SIMPLEQ_EMPTY(&th)) {
221 1.1 riastrad struct tasklet_struct *tasklet;
222 1.1 riastrad
223 1.1 riastrad /* Remove the first tasklet from the queue. */
224 1.1 riastrad tasklet = SIMPLEQ_FIRST(&th);
225 1.1 riastrad SIMPLEQ_REMOVE_HEAD(&th, tl_entry);
226 1.1 riastrad
227 1.5 riastrad KASSERT(atomic_load_relaxed(&tasklet->tl_state) &
228 1.5 riastrad TASKLET_SCHEDULED);
229 1.5 riastrad
230 1.1 riastrad /*
231 1.1 riastrad * Test and set RUNNING, in case it is already running
232 1.1 riastrad * on another CPU and got scheduled again on this one
233 1.1 riastrad * before it completed.
234 1.1 riastrad */
235 1.5 riastrad if (!tasklet_trylock(tasklet)) {
236 1.1 riastrad /*
237 1.1 riastrad * Put it back on the queue to run it again in
238 1.1 riastrad * a sort of busy-wait, and move on to the next
239 1.1 riastrad * one.
240 1.1 riastrad */
241 1.1 riastrad tasklet_queue_enqueue(tq, tasklet);
242 1.1 riastrad continue;
243 1.1 riastrad }
244 1.1 riastrad
245 1.5 riastrad /*
246 1.5 riastrad * Check whether it's currently disabled.
247 1.5 riastrad *
248 1.5 riastrad * Pairs with membar_exit in __tasklet_enable.
249 1.5 riastrad */
250 1.5 riastrad if (atomic_load_acquire(&tasklet->tl_disablecount)) {
251 1.1 riastrad /*
252 1.1 riastrad * Disabled: clear the RUNNING bit and, requeue
253 1.1 riastrad * it, but keep it SCHEDULED.
254 1.1 riastrad */
255 1.5 riastrad tasklet_unlock(tasklet);
256 1.1 riastrad tasklet_queue_enqueue(tq, tasklet);
257 1.1 riastrad continue;
258 1.1 riastrad }
259 1.1 riastrad
260 1.1 riastrad /* Not disabled. Clear SCHEDULED and call func. */
261 1.5 riastrad KASSERT(atomic_load_relaxed(&tasklet->tl_state) &
262 1.5 riastrad TASKLET_SCHEDULED);
263 1.1 riastrad atomic_and_uint(&tasklet->tl_state, ~TASKLET_SCHEDULED);
264 1.1 riastrad
265 1.1 riastrad (*tasklet->func)(tasklet->data);
266 1.1 riastrad
267 1.1 riastrad /* Clear RUNNING to notify tasklet_disable. */
268 1.5 riastrad tasklet_unlock(tasklet);
269 1.1 riastrad }
270 1.1 riastrad }
271 1.1 riastrad
272 1.1 riastrad /*
273 1.1 riastrad * tasklet_queue_schedule(tq, tasklet)
274 1.1 riastrad *
275 1.1 riastrad * Schedule tasklet to run on tq. If it was already scheduled and
276 1.1 riastrad * has not yet run, no effect.
277 1.1 riastrad */
278 1.1 riastrad static void
279 1.1 riastrad tasklet_queue_schedule(struct tasklet_queue *tq,
280 1.1 riastrad struct tasklet_struct *tasklet)
281 1.1 riastrad {
282 1.1 riastrad unsigned ostate, nstate;
283 1.1 riastrad
284 1.1 riastrad /* Test and set the SCHEDULED bit. If already set, we're done. */
285 1.1 riastrad do {
286 1.5 riastrad ostate = atomic_load_relaxed(&tasklet->tl_state);
287 1.1 riastrad if (ostate & TASKLET_SCHEDULED)
288 1.1 riastrad return;
289 1.1 riastrad nstate = ostate | TASKLET_SCHEDULED;
290 1.1 riastrad } while (atomic_cas_uint(&tasklet->tl_state, ostate, nstate)
291 1.1 riastrad != ostate);
292 1.1 riastrad
293 1.1 riastrad /*
294 1.1 riastrad * Not already set and we have set it now. Put it on the queue
295 1.1 riastrad * and kick off a softint.
296 1.1 riastrad */
297 1.1 riastrad tasklet_queue_enqueue(tq, tasklet);
298 1.1 riastrad }
299 1.1 riastrad
300 1.1 riastrad /*
301 1.1 riastrad * tasklet_queue_enqueue(tq, tasklet)
302 1.1 riastrad *
303 1.1 riastrad * Put tasklet on the queue tq and ensure it will run. tasklet
304 1.1 riastrad * must be marked SCHEDULED.
305 1.1 riastrad */
306 1.1 riastrad static void
307 1.1 riastrad tasklet_queue_enqueue(struct tasklet_queue *tq, struct tasklet_struct *tasklet)
308 1.1 riastrad {
309 1.8 riastrad struct tasklet_cpu **tcp, *tc;
310 1.1 riastrad int s;
311 1.1 riastrad
312 1.5 riastrad KASSERT(atomic_load_relaxed(&tasklet->tl_state) & TASKLET_SCHEDULED);
313 1.1 riastrad
314 1.1 riastrad /*
315 1.1 riastrad * Insert on the current CPU's queue while all interrupts are
316 1.1 riastrad * blocked, and schedule a soft interrupt to process it. No
317 1.1 riastrad * memory barriers: CPU-local state only.
318 1.1 riastrad */
319 1.8 riastrad tcp = percpu_getref(tq->tq_percpu);
320 1.8 riastrad tc = *tcp;
321 1.1 riastrad s = splhigh();
322 1.1 riastrad SIMPLEQ_INSERT_TAIL(&tc->tc_head, tasklet, tl_entry);
323 1.1 riastrad splx(s);
324 1.1 riastrad softint_schedule(tq->tq_sih);
325 1.1 riastrad percpu_putref(tq->tq_percpu);
326 1.1 riastrad }
327 1.1 riastrad
328 1.1 riastrad /*
329 1.1 riastrad * tasklet_init(tasklet, func, data)
330 1.1 riastrad *
331 1.1 riastrad * Initialize tasklet to call func(data) when scheduled.
332 1.1 riastrad *
333 1.1 riastrad * Caller is responsible for issuing the appropriate memory
334 1.1 riastrad * barriers or store releases to publish the tasklet to other CPUs
335 1.1 riastrad * before use.
336 1.1 riastrad */
337 1.1 riastrad void
338 1.1 riastrad tasklet_init(struct tasklet_struct *tasklet, void (*func)(unsigned long),
339 1.1 riastrad unsigned long data)
340 1.1 riastrad {
341 1.1 riastrad
342 1.5 riastrad atomic_store_relaxed(&tasklet->tl_state, 0);
343 1.5 riastrad atomic_store_relaxed(&tasklet->tl_disablecount, 0);
344 1.1 riastrad tasklet->func = func;
345 1.1 riastrad tasklet->data = data;
346 1.1 riastrad }
347 1.1 riastrad
348 1.1 riastrad /*
349 1.1 riastrad * tasklet_schedule(tasklet)
350 1.1 riastrad *
351 1.1 riastrad * Schedule tasklet to run at regular priority. If it was already
352 1.1 riastrad * scheduled and has not yet run, no effect.
353 1.1 riastrad */
354 1.1 riastrad void
355 1.1 riastrad tasklet_schedule(struct tasklet_struct *tasklet)
356 1.1 riastrad {
357 1.1 riastrad
358 1.1 riastrad tasklet_queue_schedule(&tasklet_queue, tasklet);
359 1.1 riastrad }
360 1.1 riastrad
361 1.1 riastrad /*
362 1.1 riastrad * tasklet_hi_schedule(tasklet)
363 1.1 riastrad *
364 1.1 riastrad * Schedule tasklet to run at high priority. If it was already
365 1.1 riastrad * scheduled and has not yet run, no effect.
366 1.1 riastrad */
367 1.1 riastrad void
368 1.1 riastrad tasklet_hi_schedule(struct tasklet_struct *tasklet)
369 1.1 riastrad {
370 1.1 riastrad
371 1.1 riastrad tasklet_queue_schedule(&tasklet_hi_queue, tasklet);
372 1.1 riastrad }
373 1.1 riastrad
374 1.1 riastrad /*
375 1.7 riastrad * tasklet_disable_nosync(tasklet)
376 1.1 riastrad *
377 1.7 riastrad * Increment the disable count of tasklet, but don't wait for it
378 1.7 riastrad * to complete -- it may remain running after this returns.
379 1.1 riastrad *
380 1.1 riastrad * As long as the disable count is nonzero, the tasklet's function
381 1.1 riastrad * will not run, but if already scheduled, the tasklet will remain
382 1.1 riastrad * so and the softint will repeatedly trigger itself in a sort of
383 1.1 riastrad * busy-wait, so this should be used only for short durations.
384 1.1 riastrad *
385 1.5 riastrad * Load-acquire semantics.
386 1.1 riastrad */
387 1.1 riastrad void
388 1.7 riastrad tasklet_disable_nosync(struct tasklet_struct *tasklet)
389 1.1 riastrad {
390 1.1 riastrad unsigned int disablecount __diagused;
391 1.1 riastrad
392 1.1 riastrad /* Increment the disable count. */
393 1.1 riastrad disablecount = atomic_inc_uint_nv(&tasklet->tl_disablecount);
394 1.1 riastrad KASSERT(disablecount < UINT_MAX);
395 1.2 riastrad KASSERT(disablecount != 0);
396 1.1 riastrad
397 1.6 riastrad /* Pairs with membar_exit in __tasklet_enable. */
398 1.6 riastrad #ifndef __HAVE_ATOMIC_AS_MEMBAR
399 1.6 riastrad membar_enter();
400 1.6 riastrad #endif
401 1.7 riastrad }
402 1.7 riastrad
403 1.7 riastrad /*
404 1.7 riastrad * tasklet_disable(tasklet)
405 1.7 riastrad *
406 1.7 riastrad * Increment the disable count of tasklet, and if it was already
407 1.7 riastrad * running, busy-wait for it to complete.
408 1.7 riastrad *
409 1.7 riastrad * As long as the disable count is nonzero, the tasklet's function
410 1.7 riastrad * will not run, but if already scheduled, the tasklet will remain
411 1.7 riastrad * so and the softint will repeatedly trigger itself in a sort of
412 1.7 riastrad * busy-wait, so this should be used only for short durations.
413 1.7 riastrad *
414 1.7 riastrad * If tasklet is guaranteed not to be scheduled, e.g. if you have
415 1.7 riastrad * just invoked tasklet_kill, then tasklet_disable serves to wait
416 1.7 riastrad * for it to complete in case it might already be running.
417 1.7 riastrad *
418 1.7 riastrad * Load-acquire semantics.
419 1.7 riastrad */
420 1.7 riastrad void
421 1.7 riastrad tasklet_disable(struct tasklet_struct *tasklet)
422 1.7 riastrad {
423 1.7 riastrad
424 1.7 riastrad /* Increment the disable count. */
425 1.7 riastrad tasklet_disable_nosync(tasklet);
426 1.6 riastrad
427 1.1 riastrad /* Wait for it to finish running, if it was running. */
428 1.5 riastrad tasklet_unlock_wait(tasklet);
429 1.1 riastrad }
430 1.1 riastrad
431 1.1 riastrad /*
432 1.1 riastrad * tasklet_enable(tasklet)
433 1.1 riastrad *
434 1.1 riastrad * Decrement tasklet's disable count. If it was previously
435 1.1 riastrad * scheduled to run, it may now run.
436 1.5 riastrad *
437 1.5 riastrad * Store-release semantics.
438 1.1 riastrad */
439 1.1 riastrad void
440 1.1 riastrad tasklet_enable(struct tasklet_struct *tasklet)
441 1.1 riastrad {
442 1.1 riastrad
443 1.5 riastrad (void)__tasklet_enable(tasklet);
444 1.1 riastrad }
445 1.1 riastrad
446 1.1 riastrad /*
447 1.1 riastrad * tasklet_kill(tasklet)
448 1.1 riastrad *
449 1.1 riastrad * Busy-wait for tasklet to run, if it is currently scheduled.
450 1.1 riastrad * Caller must guarantee it does not get scheduled again for this
451 1.1 riastrad * to be useful.
452 1.1 riastrad */
453 1.1 riastrad void
454 1.1 riastrad tasklet_kill(struct tasklet_struct *tasklet)
455 1.1 riastrad {
456 1.1 riastrad
457 1.1 riastrad KASSERTMSG(!cpu_intr_p(),
458 1.1 riastrad "deadlock: soft interrupts are blocked in interrupt context");
459 1.1 riastrad
460 1.1 riastrad /* Wait for it to be removed from the queue. */
461 1.5 riastrad while (atomic_load_relaxed(&tasklet->tl_state) & TASKLET_SCHEDULED)
462 1.1 riastrad SPINLOCK_BACKOFF_HOOK;
463 1.1 riastrad
464 1.1 riastrad /*
465 1.1 riastrad * No need for a memory barrier here because writes to the
466 1.1 riastrad * single state word are globally ordered, and RUNNING is set
467 1.1 riastrad * before SCHEDULED is cleared, so as long as the caller
468 1.1 riastrad * guarantees no scheduling, the only possible transitions we
469 1.1 riastrad * can witness are:
470 1.1 riastrad *
471 1.1 riastrad * 0 -> 0
472 1.1 riastrad * SCHEDULED -> 0
473 1.1 riastrad * SCHEDULED -> RUNNING
474 1.1 riastrad * RUNNING -> 0
475 1.1 riastrad * RUNNING -> RUNNING
476 1.1 riastrad * SCHEDULED|RUNNING -> 0
477 1.1 riastrad * SCHEDULED|RUNNING -> RUNNING
478 1.1 riastrad */
479 1.1 riastrad
480 1.1 riastrad /* Wait for it to finish running. */
481 1.5 riastrad tasklet_unlock_wait(tasklet);
482 1.5 riastrad }
483 1.5 riastrad
484 1.5 riastrad /*
485 1.10 riastrad * tasklet_is_locked(tasklet)
486 1.5 riastrad *
487 1.5 riastrad * True if tasklet is currently locked. Caller must use it only
488 1.5 riastrad * for positive assertions.
489 1.5 riastrad */
490 1.5 riastrad bool
491 1.5 riastrad tasklet_is_locked(const struct tasklet_struct *tasklet)
492 1.5 riastrad {
493 1.5 riastrad
494 1.5 riastrad return atomic_load_relaxed(&tasklet->tl_state) & TASKLET_RUNNING;
495 1.5 riastrad }
496 1.5 riastrad
497 1.5 riastrad /*
498 1.5 riastrad * tasklet_trylock(tasklet)
499 1.5 riastrad *
500 1.5 riastrad * Try to lock tasklet, i.e., set TASKLET_RUNNING. Return true if
501 1.5 riastrad * we locked it, false if already locked.
502 1.5 riastrad *
503 1.5 riastrad * Load-acquire semantics.
504 1.5 riastrad */
505 1.5 riastrad bool
506 1.5 riastrad tasklet_trylock(struct tasklet_struct *tasklet)
507 1.5 riastrad {
508 1.5 riastrad unsigned state;
509 1.5 riastrad
510 1.5 riastrad do {
511 1.6 riastrad state = atomic_load_relaxed(&tasklet->tl_state);
512 1.5 riastrad if (state & TASKLET_RUNNING)
513 1.5 riastrad return false;
514 1.5 riastrad } while (atomic_cas_uint(&tasklet->tl_state, state,
515 1.5 riastrad state | TASKLET_RUNNING) != state);
516 1.5 riastrad
517 1.6 riastrad /* Pairs with membar_exit in tasklet_unlock. */
518 1.6 riastrad #ifndef __HAVE_ATOMIC_AS_MEMBAR
519 1.6 riastrad membar_enter();
520 1.6 riastrad #endif
521 1.6 riastrad
522 1.5 riastrad return true;
523 1.5 riastrad }
524 1.5 riastrad
525 1.5 riastrad /*
526 1.5 riastrad * tasklet_unlock(tasklet)
527 1.5 riastrad *
528 1.5 riastrad * Unlock tasklet, i.e., clear TASKLET_RUNNING.
529 1.5 riastrad *
530 1.5 riastrad * Store-release semantics.
531 1.5 riastrad */
532 1.5 riastrad void
533 1.5 riastrad tasklet_unlock(struct tasklet_struct *tasklet)
534 1.5 riastrad {
535 1.5 riastrad
536 1.5 riastrad KASSERT(atomic_load_relaxed(&tasklet->tl_state) & TASKLET_RUNNING);
537 1.1 riastrad
538 1.1 riastrad /*
539 1.6 riastrad * Pairs with membar_enter in tasklet_trylock and with
540 1.6 riastrad * atomic_load_acquire in tasklet_unlock_wait.
541 1.1 riastrad */
542 1.5 riastrad #ifndef __HAVE_ATOMIC_AS_MEMBAR
543 1.5 riastrad membar_exit();
544 1.5 riastrad #endif
545 1.5 riastrad atomic_and_uint(&tasklet->tl_state, ~TASKLET_RUNNING);
546 1.5 riastrad }
547 1.5 riastrad
548 1.5 riastrad /*
549 1.5 riastrad * tasklet_unlock_wait(tasklet)
550 1.5 riastrad *
551 1.5 riastrad * Busy-wait until tasklet is not running.
552 1.5 riastrad *
553 1.5 riastrad * Load-acquire semantics.
554 1.5 riastrad */
555 1.5 riastrad void
556 1.5 riastrad tasklet_unlock_wait(const struct tasklet_struct *tasklet)
557 1.5 riastrad {
558 1.5 riastrad
559 1.5 riastrad /* Pairs with membar_exit in tasklet_unlock. */
560 1.5 riastrad while (atomic_load_acquire(&tasklet->tl_state) & TASKLET_RUNNING)
561 1.5 riastrad SPINLOCK_BACKOFF_HOOK;
562 1.1 riastrad }
563 1.3 riastrad
564 1.3 riastrad /*
565 1.5 riastrad * BEGIN I915 HACKS
566 1.5 riastrad *
567 1.5 riastrad * The i915 driver abuses the tasklet abstraction like a cop abuses his
568 1.5 riastrad * wife.
569 1.5 riastrad */
570 1.5 riastrad
571 1.5 riastrad /*
572 1.5 riastrad * __tasklet_disable_sync_once(tasklet)
573 1.3 riastrad *
574 1.3 riastrad * Increment the disable count of tasklet, and if this is the
575 1.3 riastrad * first time it was disabled and it was already running,
576 1.3 riastrad * busy-wait for it to complete.
577 1.3 riastrad *
578 1.3 riastrad * Caller must not care about whether the tasklet is running, or
579 1.3 riastrad * about waiting for any side effects of the tasklet to complete,
580 1.3 riastrad * if this was not the first time it was disabled.
581 1.3 riastrad */
582 1.3 riastrad void
583 1.5 riastrad __tasklet_disable_sync_once(struct tasklet_struct *tasklet)
584 1.3 riastrad {
585 1.3 riastrad unsigned int disablecount;
586 1.3 riastrad
587 1.3 riastrad /* Increment the disable count. */
588 1.3 riastrad disablecount = atomic_inc_uint_nv(&tasklet->tl_disablecount);
589 1.3 riastrad KASSERT(disablecount < UINT_MAX);
590 1.3 riastrad KASSERT(disablecount != 0);
591 1.3 riastrad
592 1.6 riastrad /* Pairs with membar_exit in __tasklet_enable_sync_once. */
593 1.6 riastrad #ifndef __HAVE_ATOMIC_AS_MEMBAR
594 1.6 riastrad membar_enter();
595 1.6 riastrad #endif
596 1.6 riastrad
597 1.3 riastrad /*
598 1.3 riastrad * If it was zero, wait for it to finish running. If it was
599 1.3 riastrad * not zero, caller must not care whether it was running.
600 1.3 riastrad */
601 1.5 riastrad if (disablecount == 1)
602 1.5 riastrad tasklet_unlock_wait(tasklet);
603 1.3 riastrad }
604 1.3 riastrad
605 1.3 riastrad /*
606 1.5 riastrad * __tasklet_enable_sync_once(tasklet)
607 1.3 riastrad *
608 1.3 riastrad * Decrement the disable count of tasklet, and if it goes to zero,
609 1.3 riastrad * kill tasklet.
610 1.3 riastrad */
611 1.3 riastrad void
612 1.5 riastrad __tasklet_enable_sync_once(struct tasklet_struct *tasklet)
613 1.3 riastrad {
614 1.3 riastrad unsigned int disablecount;
615 1.3 riastrad
616 1.6 riastrad /* Pairs with membar_enter in __tasklet_disable_sync_once. */
617 1.6 riastrad #ifndef __HAVE_ATOMIC_AS_MEMBAR
618 1.6 riastrad membar_exit();
619 1.6 riastrad #endif
620 1.6 riastrad
621 1.3 riastrad /* Decrement the disable count. */
622 1.3 riastrad disablecount = atomic_dec_uint_nv(&tasklet->tl_disablecount);
623 1.3 riastrad KASSERT(disablecount < UINT_MAX);
624 1.3 riastrad
625 1.3 riastrad /*
626 1.3 riastrad * If it became zero, kill the tasklet. If it was not zero,
627 1.3 riastrad * caller must not care whether it was running.
628 1.3 riastrad */
629 1.3 riastrad if (disablecount == 0)
630 1.3 riastrad tasklet_kill(tasklet);
631 1.3 riastrad }
632 1.3 riastrad
633 1.3 riastrad /*
634 1.5 riastrad * __tasklet_is_enabled(tasklet)
635 1.3 riastrad *
636 1.3 riastrad * True if tasklet is not currently disabled. Answer may be stale
637 1.3 riastrad * as soon as it is returned -- caller must use it only as a hint,
638 1.3 riastrad * or must arrange synchronization externally.
639 1.3 riastrad */
640 1.3 riastrad bool
641 1.5 riastrad __tasklet_is_enabled(const struct tasklet_struct *tasklet)
642 1.3 riastrad {
643 1.3 riastrad unsigned int disablecount;
644 1.3 riastrad
645 1.5 riastrad disablecount = atomic_load_relaxed(&tasklet->tl_disablecount);
646 1.5 riastrad
647 1.5 riastrad return (disablecount == 0);
648 1.5 riastrad }
649 1.5 riastrad
650 1.5 riastrad /*
651 1.5 riastrad * __tasklet_is_scheduled(tasklet)
652 1.5 riastrad *
653 1.5 riastrad * True if tasklet is currently scheduled. Answer may be stale as
654 1.5 riastrad * soon as it is returned -- caller must use it only as a hint, or
655 1.5 riastrad * must arrange synchronization externally.
656 1.5 riastrad */
657 1.5 riastrad bool
658 1.5 riastrad __tasklet_is_scheduled(const struct tasklet_struct *tasklet)
659 1.5 riastrad {
660 1.5 riastrad
661 1.5 riastrad return atomic_load_relaxed(&tasklet->tl_state) & TASKLET_SCHEDULED;
662 1.5 riastrad }
663 1.5 riastrad
664 1.5 riastrad /*
665 1.5 riastrad * __tasklet_enable(tasklet)
666 1.5 riastrad *
667 1.5 riastrad * Decrement tasklet's disable count. If it was previously
668 1.5 riastrad * scheduled to run, it may now run. Return true if the disable
669 1.5 riastrad * count went down to zero; otherwise return false.
670 1.5 riastrad *
671 1.5 riastrad * Store-release semantics.
672 1.5 riastrad */
673 1.5 riastrad bool
674 1.5 riastrad __tasklet_enable(struct tasklet_struct *tasklet)
675 1.5 riastrad {
676 1.5 riastrad unsigned int disablecount;
677 1.5 riastrad
678 1.5 riastrad /*
679 1.5 riastrad * Guarantee all caller-relevant reads or writes have completed
680 1.5 riastrad * before potentially allowing tasklet to run again by
681 1.5 riastrad * decrementing the disable count.
682 1.5 riastrad *
683 1.6 riastrad * Pairs with atomic_load_acquire in tasklet_softintr and with
684 1.6 riastrad * membar_enter in tasklet_disable.
685 1.5 riastrad */
686 1.5 riastrad #ifndef __HAVE_ATOMIC_AS_MEMBAR
687 1.5 riastrad membar_exit();
688 1.5 riastrad #endif
689 1.5 riastrad
690 1.5 riastrad /* Decrement the disable count. */
691 1.5 riastrad disablecount = atomic_dec_uint_nv(&tasklet->tl_disablecount);
692 1.5 riastrad KASSERT(disablecount != UINT_MAX);
693 1.3 riastrad
694 1.3 riastrad return (disablecount == 0);
695 1.3 riastrad }
696