linux_work.c revision 1.1.24.1 1 1.1.24.1 pgoyette /* $NetBSD: linux_work.c,v 1.1.24.1 2018/09/06 06:56:08 pgoyette Exp $ */
2 1.1 skrll
3 1.1 skrll /*-
4 1.1.24.1 pgoyette * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 1.1 skrll * All rights reserved.
6 1.1 skrll *
7 1.1 skrll * This code is derived from software contributed to The NetBSD Foundation
8 1.1 skrll * by Taylor R. Campbell.
9 1.1 skrll *
10 1.1 skrll * Redistribution and use in source and binary forms, with or without
11 1.1 skrll * modification, are permitted provided that the following conditions
12 1.1 skrll * are met:
13 1.1 skrll * 1. Redistributions of source code must retain the above copyright
14 1.1 skrll * notice, this list of conditions and the following disclaimer.
15 1.1 skrll * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 skrll * notice, this list of conditions and the following disclaimer in the
17 1.1 skrll * documentation and/or other materials provided with the distribution.
18 1.1 skrll *
19 1.1 skrll * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.1 skrll * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.1 skrll * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.1 skrll * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.1 skrll * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.1 skrll * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.1 skrll * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.1 skrll * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.1 skrll * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.1 skrll * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 skrll * POSSIBILITY OF SUCH DAMAGE.
30 1.1 skrll */
31 1.1 skrll
32 1.1 skrll #include <sys/cdefs.h>
33 1.1.24.1 pgoyette __KERNEL_RCSID(0, "$NetBSD: linux_work.c,v 1.1.24.1 2018/09/06 06:56:08 pgoyette Exp $");
34 1.1 skrll
35 1.1 skrll #include <sys/types.h>
36 1.1 skrll #include <sys/atomic.h>
37 1.1 skrll #include <sys/callout.h>
38 1.1 skrll #include <sys/condvar.h>
39 1.1 skrll #include <sys/errno.h>
40 1.1 skrll #include <sys/kmem.h>
41 1.1.24.1 pgoyette #include <sys/kthread.h>
42 1.1.24.1 pgoyette #include <sys/lwp.h>
43 1.1 skrll #include <sys/mutex.h>
44 1.1 skrll #include <sys/queue.h>
45 1.1.24.1 pgoyette #include <sys/sdt.h>
46 1.1 skrll
47 1.1 skrll #include <linux/workqueue.h>
48 1.1 skrll
49 1.1.24.1 pgoyette TAILQ_HEAD(work_head, work_struct);
50 1.1.24.1 pgoyette TAILQ_HEAD(dwork_head, delayed_work);
51 1.1 skrll
52 1.1 skrll struct workqueue_struct {
53 1.1.24.1 pgoyette kmutex_t wq_lock;
54 1.1.24.1 pgoyette kcondvar_t wq_cv;
55 1.1.24.1 pgoyette struct dwork_head wq_delayed; /* delayed work scheduled */
56 1.1.24.1 pgoyette struct work_head wq_queue; /* work to run */
57 1.1.24.1 pgoyette struct work_head wq_dqueue; /* delayed work to run now */
58 1.1.24.1 pgoyette struct work_struct *wq_current_work;
59 1.1.24.1 pgoyette int wq_flags;
60 1.1.24.1 pgoyette bool wq_dying;
61 1.1.24.1 pgoyette uint64_t wq_gen;
62 1.1.24.1 pgoyette struct lwp *wq_lwp;
63 1.1 skrll };
64 1.1 skrll
65 1.1.24.1 pgoyette static void __dead linux_workqueue_thread(void *);
66 1.1.24.1 pgoyette static void linux_workqueue_timeout(void *);
67 1.1.24.1 pgoyette static bool work_claimed(struct work_struct *,
68 1.1.24.1 pgoyette struct workqueue_struct *);
69 1.1.24.1 pgoyette static struct workqueue_struct *
70 1.1.24.1 pgoyette work_queue(struct work_struct *);
71 1.1.24.1 pgoyette static bool acquire_work(struct work_struct *,
72 1.1.24.1 pgoyette struct workqueue_struct *);
73 1.1.24.1 pgoyette static void release_work(struct work_struct *,
74 1.1.24.1 pgoyette struct workqueue_struct *);
75 1.1.24.1 pgoyette static void wait_for_current_work(struct work_struct *,
76 1.1.24.1 pgoyette struct workqueue_struct *);
77 1.1.24.1 pgoyette static void dw_callout_init(struct workqueue_struct *,
78 1.1.24.1 pgoyette struct delayed_work *);
79 1.1.24.1 pgoyette static void dw_callout_destroy(struct workqueue_struct *,
80 1.1.24.1 pgoyette struct delayed_work *);
81 1.1.24.1 pgoyette static void cancel_delayed_work_done(struct workqueue_struct *,
82 1.1.24.1 pgoyette struct delayed_work *);
83 1.1.24.1 pgoyette
84 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, acquire,
85 1.1.24.1 pgoyette "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
86 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, release,
87 1.1.24.1 pgoyette "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
88 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, queue,
89 1.1.24.1 pgoyette "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
90 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, cancel,
91 1.1.24.1 pgoyette "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
92 1.1.24.1 pgoyette SDT_PROBE_DEFINE3(sdt, linux, work, schedule,
93 1.1.24.1 pgoyette "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/,
94 1.1.24.1 pgoyette "unsigned long"/*ticks*/);
95 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, timer,
96 1.1.24.1 pgoyette "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/);
97 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, wait__start,
98 1.1.24.1 pgoyette "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/);
99 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, wait__done,
100 1.1.24.1 pgoyette "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/);
101 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, run,
102 1.1.24.1 pgoyette "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
103 1.1.24.1 pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, done,
104 1.1.24.1 pgoyette "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
105 1.1.24.1 pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, batch__start,
106 1.1.24.1 pgoyette "struct workqueue_struct *"/*wq*/);
107 1.1.24.1 pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, batch__done,
108 1.1.24.1 pgoyette "struct workqueue_struct *"/*wq*/);
109 1.1.24.1 pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, flush__start,
110 1.1.24.1 pgoyette "struct workqueue_struct *"/*wq*/);
111 1.1.24.1 pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, flush__done,
112 1.1.24.1 pgoyette "struct workqueue_struct *"/*wq*/);
113 1.1.24.1 pgoyette
114 1.1.24.1 pgoyette static specificdata_key_t workqueue_key __read_mostly;
115 1.1.24.1 pgoyette
116 1.1.24.1 pgoyette struct workqueue_struct *system_wq __read_mostly;
117 1.1.24.1 pgoyette struct workqueue_struct *system_long_wq __read_mostly;
118 1.1.24.1 pgoyette struct workqueue_struct *system_power_efficient_wq __read_mostly;
119 1.1 skrll
120 1.1.24.1 pgoyette static inline uintptr_t
121 1.1.24.1 pgoyette atomic_cas_uintptr(volatile uintptr_t *p, uintptr_t old, uintptr_t new)
122 1.1.24.1 pgoyette {
123 1.1 skrll
124 1.1.24.1 pgoyette return (uintptr_t)atomic_cas_ptr(p, (void *)old, (void *)new);
125 1.1.24.1 pgoyette }
126 1.1.24.1 pgoyette
127 1.1.24.1 pgoyette /*
128 1.1.24.1 pgoyette * linux_workqueue_init()
129 1.1.24.1 pgoyette *
130 1.1.24.1 pgoyette * Initialize the Linux workqueue subsystem. Return 0 on success,
131 1.1.24.1 pgoyette * NetBSD error on failure.
132 1.1.24.1 pgoyette */
133 1.1 skrll int
134 1.1 skrll linux_workqueue_init(void)
135 1.1 skrll {
136 1.1.24.1 pgoyette int error;
137 1.1.24.1 pgoyette
138 1.1.24.1 pgoyette error = lwp_specific_key_create(&workqueue_key, NULL);
139 1.1.24.1 pgoyette if (error)
140 1.1.24.1 pgoyette goto fail0;
141 1.1 skrll
142 1.1 skrll system_wq = alloc_ordered_workqueue("lnxsyswq", 0);
143 1.1.24.1 pgoyette if (system_wq == NULL) {
144 1.1.24.1 pgoyette error = ENOMEM;
145 1.1.24.1 pgoyette goto fail1;
146 1.1.24.1 pgoyette }
147 1.1.24.1 pgoyette
148 1.1.24.1 pgoyette system_long_wq = alloc_ordered_workqueue("lnxlngwq", 0);
149 1.1.24.1 pgoyette if (system_long_wq == NULL) {
150 1.1.24.1 pgoyette error = ENOMEM;
151 1.1.24.1 pgoyette goto fail2;
152 1.1.24.1 pgoyette }
153 1.1.24.1 pgoyette
154 1.1.24.1 pgoyette system_power_efficient_wq = alloc_ordered_workqueue("lnxpwrwq", 0);
155 1.1.24.1 pgoyette if (system_long_wq == NULL) {
156 1.1.24.1 pgoyette error = ENOMEM;
157 1.1.24.1 pgoyette goto fail3;
158 1.1.24.1 pgoyette }
159 1.1 skrll
160 1.1 skrll return 0;
161 1.1.24.1 pgoyette
162 1.1.24.1 pgoyette fail4: __unused
163 1.1.24.1 pgoyette destroy_workqueue(system_power_efficient_wq);
164 1.1.24.1 pgoyette fail3: destroy_workqueue(system_long_wq);
165 1.1.24.1 pgoyette fail2: destroy_workqueue(system_wq);
166 1.1.24.1 pgoyette fail1: lwp_specific_key_delete(workqueue_key);
167 1.1.24.1 pgoyette fail0: KASSERT(error);
168 1.1.24.1 pgoyette return error;
169 1.1 skrll }
170 1.1 skrll
171 1.1.24.1 pgoyette /*
172 1.1.24.1 pgoyette * linux_workqueue_fini()
173 1.1.24.1 pgoyette *
174 1.1.24.1 pgoyette * Destroy the Linux workqueue subsystem. Never fails.
175 1.1.24.1 pgoyette */
176 1.1 skrll void
177 1.1 skrll linux_workqueue_fini(void)
178 1.1 skrll {
179 1.1.24.1 pgoyette
180 1.1.24.1 pgoyette destroy_workqueue(system_power_efficient_wq);
181 1.1.24.1 pgoyette destroy_workqueue(system_long_wq);
182 1.1 skrll destroy_workqueue(system_wq);
183 1.1.24.1 pgoyette lwp_specific_key_delete(workqueue_key);
184 1.1 skrll }
185 1.1 skrll
186 1.1 skrll /*
188 1.1 skrll * Workqueues
189 1.1 skrll */
190 1.1.24.1 pgoyette
191 1.1.24.1 pgoyette /*
192 1.1.24.1 pgoyette * alloc_ordered_workqueue(name, flags)
193 1.1.24.1 pgoyette *
194 1.1.24.1 pgoyette * Create a workqueue of the given name. No flags are currently
195 1.1.24.1 pgoyette * defined. Return NULL on failure, pointer to struct
196 1.1.24.1 pgoyette * workqueue_struct object on success.
197 1.1 skrll */
198 1.1.24.1 pgoyette struct workqueue_struct *
199 1.1 skrll alloc_ordered_workqueue(const char *name, int flags)
200 1.1 skrll {
201 1.1 skrll struct workqueue_struct *wq;
202 1.1 skrll int error;
203 1.1.24.1 pgoyette
204 1.1 skrll KASSERT(flags == 0);
205 1.1.24.1 pgoyette
206 1.1 skrll wq = kmem_zalloc(sizeof(*wq), KM_SLEEP);
207 1.1 skrll
208 1.1 skrll mutex_init(&wq->wq_lock, MUTEX_DEFAULT, IPL_VM);
209 1.1 skrll cv_init(&wq->wq_cv, name);
210 1.1.24.1 pgoyette TAILQ_INIT(&wq->wq_delayed);
211 1.1.24.1 pgoyette TAILQ_INIT(&wq->wq_queue);
212 1.1 skrll TAILQ_INIT(&wq->wq_dqueue);
213 1.1.24.1 pgoyette wq->wq_current_work = NULL;
214 1.1.24.1 pgoyette wq->wq_flags = 0;
215 1.1.24.1 pgoyette wq->wq_dying = false;
216 1.1.24.1 pgoyette wq->wq_gen = 0;
217 1.1.24.1 pgoyette wq->wq_lwp = NULL;
218 1.1.24.1 pgoyette
219 1.1.24.1 pgoyette error = kthread_create(PRI_NONE,
220 1.1.24.1 pgoyette KTHREAD_MPSAFE|KTHREAD_TS|KTHREAD_MUSTJOIN, NULL,
221 1.1.24.1 pgoyette &linux_workqueue_thread, wq, &wq->wq_lwp, "%s", name);
222 1.1.24.1 pgoyette if (error)
223 1.1 skrll goto fail0;
224 1.1 skrll
225 1.1.24.1 pgoyette return wq;
226 1.1.24.1 pgoyette
227 1.1.24.1 pgoyette fail0: KASSERT(TAILQ_EMPTY(&wq->wq_dqueue));
228 1.1.24.1 pgoyette KASSERT(TAILQ_EMPTY(&wq->wq_queue));
229 1.1.24.1 pgoyette KASSERT(TAILQ_EMPTY(&wq->wq_delayed));
230 1.1.24.1 pgoyette cv_destroy(&wq->wq_cv);
231 1.1.24.1 pgoyette mutex_destroy(&wq->wq_lock);
232 1.1.24.1 pgoyette kmem_free(wq, sizeof(*wq));
233 1.1 skrll return NULL;
234 1.1 skrll }
235 1.1.24.1 pgoyette
236 1.1.24.1 pgoyette /*
237 1.1.24.1 pgoyette * destroy_workqueue(wq)
238 1.1.24.1 pgoyette *
239 1.1.24.1 pgoyette * Destroy a workqueue created with wq. Cancel any pending
240 1.1.24.1 pgoyette * delayed work. Wait for all queued work to complete.
241 1.1.24.1 pgoyette *
242 1.1.24.1 pgoyette * May sleep.
243 1.1 skrll */
244 1.1 skrll void
245 1.1 skrll destroy_workqueue(struct workqueue_struct *wq)
246 1.1 skrll {
247 1.1 skrll
248 1.1.24.1 pgoyette /*
249 1.1.24.1 pgoyette * Cancel all delayed work. We do this first because any
250 1.1.24.1 pgoyette * delayed work that that has already timed out, which we can't
251 1.1 skrll * cancel, may have queued new work.
252 1.1.24.1 pgoyette */
253 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
254 1.1.24.1 pgoyette while (!TAILQ_EMPTY(&wq->wq_delayed)) {
255 1.1 skrll struct delayed_work *const dw = TAILQ_FIRST(&wq->wq_delayed);
256 1.1.24.1 pgoyette
257 1.1.24.1 pgoyette KASSERT(work_queue(&dw->work) == wq);
258 1.1.24.1 pgoyette KASSERTMSG((dw->dw_state == DELAYED_WORK_SCHEDULED ||
259 1.1.24.1 pgoyette dw->dw_state == DELAYED_WORK_RESCHEDULED ||
260 1.1.24.1 pgoyette dw->dw_state == DELAYED_WORK_CANCELLED),
261 1.1.24.1 pgoyette "delayed work %p in bad state: %d",
262 1.1 skrll dw, dw->dw_state);
263 1.1.24.1 pgoyette
264 1.1.24.1 pgoyette /*
265 1.1.24.1 pgoyette * Mark it cancelled and try to stop the callout before
266 1.1.24.1 pgoyette * it starts.
267 1.1.24.1 pgoyette *
268 1.1.24.1 pgoyette * If it's too late and the callout has already begun
269 1.1.24.1 pgoyette * to execute, then it will notice that we asked to
270 1.1.24.1 pgoyette * cancel it and remove itself from the queue before
271 1.1.24.1 pgoyette * returning.
272 1.1.24.1 pgoyette *
273 1.1.24.1 pgoyette * If we stopped the callout before it started,
274 1.1.24.1 pgoyette * however, then we can safely destroy the callout and
275 1.1.24.1 pgoyette * dissociate it from the workqueue ourselves.
276 1.1.24.1 pgoyette */
277 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq);
278 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_CANCELLED;
279 1.1.24.1 pgoyette if (!callout_halt(&dw->dw_callout, &wq->wq_lock))
280 1.1 skrll cancel_delayed_work_done(wq, dw);
281 1.1.24.1 pgoyette }
282 1.1 skrll mutex_exit(&wq->wq_lock);
283 1.1 skrll
284 1.1.24.1 pgoyette /*
285 1.1 skrll * At this point, no new work can be put on the queue.
286 1.1 skrll */
287 1.1.24.1 pgoyette
288 1.1.24.1 pgoyette /* Tell the thread to exit. */
289 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
290 1.1.24.1 pgoyette wq->wq_dying = true;
291 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
292 1.1.24.1 pgoyette mutex_exit(&wq->wq_lock);
293 1.1.24.1 pgoyette
294 1.1.24.1 pgoyette /* Wait for it to exit. */
295 1.1.24.1 pgoyette (void)kthread_join(wq->wq_lwp);
296 1.1.24.1 pgoyette
297 1.1.24.1 pgoyette KASSERT(wq->wq_dying);
298 1.1.24.1 pgoyette KASSERT(wq->wq_flags == 0);
299 1.1.24.1 pgoyette KASSERT(wq->wq_current_work == NULL);
300 1.1.24.1 pgoyette KASSERT(TAILQ_EMPTY(&wq->wq_dqueue));
301 1.1.24.1 pgoyette KASSERT(TAILQ_EMPTY(&wq->wq_queue));
302 1.1 skrll KASSERT(TAILQ_EMPTY(&wq->wq_delayed));
303 1.1 skrll cv_destroy(&wq->wq_cv);
304 1.1 skrll mutex_destroy(&wq->wq_lock);
305 1.1 skrll
306 1.1 skrll kmem_free(wq, sizeof(*wq));
307 1.1 skrll }
308 1.1 skrll
309 1.1.24.1 pgoyette /*
311 1.1 skrll * Work thread and callout
312 1.1.24.1 pgoyette */
313 1.1.24.1 pgoyette
314 1.1.24.1 pgoyette /*
315 1.1.24.1 pgoyette * linux_workqueue_thread(cookie)
316 1.1.24.1 pgoyette *
317 1.1.24.1 pgoyette * Main function for a workqueue's worker thread. Waits until
318 1.1.24.1 pgoyette * there is work queued, grabs a batch of work off the queue,
319 1.1.24.1 pgoyette * executes it all, bumps the generation number, and repeats,
320 1.1.24.1 pgoyette * until dying.
321 1.1.24.1 pgoyette */
322 1.1 skrll static void __dead
323 1.1.24.1 pgoyette linux_workqueue_thread(void *cookie)
324 1.1.24.1 pgoyette {
325 1.1.24.1 pgoyette struct workqueue_struct *const wq = cookie;
326 1.1.24.1 pgoyette struct work_head queue, dqueue;
327 1.1 skrll struct work_head *const q[2] = { &queue, &dqueue };
328 1.1.24.1 pgoyette unsigned i;
329 1.1 skrll
330 1.1.24.1 pgoyette lwp_setspecific(workqueue_key, wq);
331 1.1.24.1 pgoyette
332 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
333 1.1.24.1 pgoyette for (;;) {
334 1.1.24.1 pgoyette /*
335 1.1.24.1 pgoyette * Wait until there's activity. If there's no work and
336 1.1.24.1 pgoyette * we're dying, stop here.
337 1.1.24.1 pgoyette */
338 1.1.24.1 pgoyette if (TAILQ_EMPTY(&wq->wq_queue) &&
339 1.1.24.1 pgoyette TAILQ_EMPTY(&wq->wq_dqueue)) {
340 1.1.24.1 pgoyette if (wq->wq_dying)
341 1.1.24.1 pgoyette break;
342 1.1.24.1 pgoyette cv_wait(&wq->wq_cv, &wq->wq_lock);
343 1.1 skrll continue;
344 1.1.24.1 pgoyette }
345 1.1.24.1 pgoyette
346 1.1.24.1 pgoyette /* Grab a batch of work off the queue. */
347 1.1.24.1 pgoyette SDT_PROBE1(sdt, linux, work, batch__start, wq);
348 1.1.24.1 pgoyette TAILQ_INIT(&queue);
349 1.1.24.1 pgoyette TAILQ_INIT(&dqueue);
350 1.1.24.1 pgoyette TAILQ_CONCAT(&queue, &wq->wq_queue, work_entry);
351 1.1.24.1 pgoyette TAILQ_CONCAT(&dqueue, &wq->wq_dqueue, work_entry);
352 1.1.24.1 pgoyette
353 1.1.24.1 pgoyette /* Process each work item in the batch. */
354 1.1.24.1 pgoyette for (i = 0; i < 2; i++) {
355 1.1.24.1 pgoyette while (!TAILQ_EMPTY(q[i])) {
356 1.1.24.1 pgoyette struct work_struct *work = TAILQ_FIRST(q[i]);
357 1.1.24.1 pgoyette void (*func)(struct work_struct *);
358 1.1.24.1 pgoyette
359 1.1.24.1 pgoyette KASSERT(work_queue(work) == wq);
360 1.1.24.1 pgoyette KASSERT(work_claimed(work, wq));
361 1.1.24.1 pgoyette KASSERTMSG((q[i] != &dqueue ||
362 1.1.24.1 pgoyette container_of(work, struct delayed_work,
363 1.1.24.1 pgoyette work)->dw_state ==
364 1.1.24.1 pgoyette DELAYED_WORK_IDLE),
365 1.1.24.1 pgoyette "delayed work %p queued and scheduled",
366 1.1.24.1 pgoyette work);
367 1.1.24.1 pgoyette
368 1.1.24.1 pgoyette TAILQ_REMOVE(q[i], work, work_entry);
369 1.1.24.1 pgoyette KASSERT(wq->wq_current_work == NULL);
370 1.1.24.1 pgoyette wq->wq_current_work = work;
371 1.1.24.1 pgoyette func = work->func;
372 1.1.24.1 pgoyette release_work(work, wq);
373 1.1.24.1 pgoyette /* Can't dereference work after this point. */
374 1.1.24.1 pgoyette
375 1.1.24.1 pgoyette mutex_exit(&wq->wq_lock);
376 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, run, work, wq);
377 1.1.24.1 pgoyette (*func)(work);
378 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, done, work, wq);
379 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
380 1.1.24.1 pgoyette
381 1.1.24.1 pgoyette KASSERT(wq->wq_current_work == work);
382 1.1.24.1 pgoyette wq->wq_current_work = NULL;
383 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
384 1.1 skrll }
385 1.1.24.1 pgoyette }
386 1.1.24.1 pgoyette
387 1.1.24.1 pgoyette /* Notify flush that we've completed a batch of work. */
388 1.1.24.1 pgoyette wq->wq_gen++;
389 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
390 1.1.24.1 pgoyette SDT_PROBE1(sdt, linux, work, batch__done, wq);
391 1.1.24.1 pgoyette }
392 1.1.24.1 pgoyette mutex_exit(&wq->wq_lock);
393 1.1 skrll
394 1.1 skrll kthread_exit(0);
395 1.1.24.1 pgoyette }
396 1.1.24.1 pgoyette
397 1.1.24.1 pgoyette /*
398 1.1.24.1 pgoyette * linux_workqueue_timeout(cookie)
399 1.1.24.1 pgoyette *
400 1.1.24.1 pgoyette * Delayed work timeout callback.
401 1.1.24.1 pgoyette *
402 1.1.24.1 pgoyette * - If scheduled, queue it.
403 1.1.24.1 pgoyette * - If rescheduled, callout_schedule ourselves again.
404 1.1.24.1 pgoyette * - If cancelled, destroy the callout and release the work from
405 1.1.24.1 pgoyette * the workqueue.
406 1.1.24.1 pgoyette */
407 1.1 skrll static void
408 1.1.24.1 pgoyette linux_workqueue_timeout(void *cookie)
409 1.1.24.1 pgoyette {
410 1.1 skrll struct delayed_work *const dw = cookie;
411 1.1.24.1 pgoyette struct workqueue_struct *const wq = work_queue(&dw->work);
412 1.1.24.1 pgoyette
413 1.1.24.1 pgoyette KASSERTMSG(wq != NULL,
414 1.1 skrll "delayed work %p state %d resched %d",
415 1.1.24.1 pgoyette dw, dw->dw_state, dw->dw_resched);
416 1.1 skrll
417 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, timer, dw, wq);
418 1.1.24.1 pgoyette
419 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
420 1.1.24.1 pgoyette KASSERT(work_queue(&dw->work) == wq);
421 1.1.24.1 pgoyette switch (dw->dw_state) {
422 1.1.24.1 pgoyette case DELAYED_WORK_IDLE:
423 1.1.24.1 pgoyette panic("delayed work callout uninitialized: %p", dw);
424 1.1.24.1 pgoyette case DELAYED_WORK_SCHEDULED:
425 1.1.24.1 pgoyette dw_callout_destroy(wq, dw);
426 1.1.24.1 pgoyette TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work, work_entry);
427 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
428 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue, &dw->work, wq);
429 1.1.24.1 pgoyette break;
430 1.1.24.1 pgoyette case DELAYED_WORK_RESCHEDULED:
431 1.1.24.1 pgoyette KASSERT(dw->dw_resched >= 0);
432 1.1.24.1 pgoyette callout_schedule(&dw->dw_callout, dw->dw_resched);
433 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_SCHEDULED;
434 1.1.24.1 pgoyette dw->dw_resched = -1;
435 1.1.24.1 pgoyette break;
436 1.1.24.1 pgoyette case DELAYED_WORK_CANCELLED:
437 1.1.24.1 pgoyette cancel_delayed_work_done(wq, dw);
438 1.1.24.1 pgoyette /* Can't dereference dw after this point. */
439 1.1.24.1 pgoyette goto out;
440 1.1.24.1 pgoyette default:
441 1.1.24.1 pgoyette panic("delayed work callout in bad state: %p", dw);
442 1.1.24.1 pgoyette }
443 1.1.24.1 pgoyette KASSERT(dw->dw_state == DELAYED_WORK_IDLE ||
444 1.1 skrll dw->dw_state == DELAYED_WORK_SCHEDULED);
445 1.1 skrll out: mutex_exit(&wq->wq_lock);
446 1.1.24.1 pgoyette }
447 1.1.24.1 pgoyette
448 1.1.24.1 pgoyette /*
449 1.1.24.1 pgoyette * current_work()
450 1.1.24.1 pgoyette *
451 1.1.24.1 pgoyette * If in a workqueue worker thread, return the work it is
452 1.1.24.1 pgoyette * currently executing. Otherwise return NULL.
453 1.1.24.1 pgoyette */
454 1.1 skrll struct work_struct *
455 1.1.24.1 pgoyette current_work(void)
456 1.1 skrll {
457 1.1.24.1 pgoyette struct workqueue_struct *wq = lwp_getspecific(workqueue_key);
458 1.1.24.1 pgoyette
459 1.1.24.1 pgoyette /* If we're not a workqueue thread, then there's no work. */
460 1.1.24.1 pgoyette if (wq == NULL)
461 1.1.24.1 pgoyette return NULL;
462 1.1.24.1 pgoyette
463 1.1.24.1 pgoyette /*
464 1.1.24.1 pgoyette * Otherwise, this should be possible only while work is in
465 1.1.24.1 pgoyette * progress. Return the current work item.
466 1.1.24.1 pgoyette */
467 1.1 skrll KASSERT(wq->wq_current_work != NULL);
468 1.1 skrll return wq->wq_current_work;
469 1.1 skrll }
470 1.1.24.1 pgoyette
471 1.1 skrll /*
473 1.1.24.1 pgoyette * Work
474 1.1.24.1 pgoyette */
475 1.1.24.1 pgoyette
476 1.1.24.1 pgoyette /*
477 1.1.24.1 pgoyette * INIT_WORK(work, fn)
478 1.1.24.1 pgoyette *
479 1.1.24.1 pgoyette * Initialize work for use with a workqueue to call fn in a worker
480 1.1.24.1 pgoyette * thread. There is no corresponding destruction operation.
481 1.1 skrll */
482 1.1 skrll void
483 1.1.24.1 pgoyette INIT_WORK(struct work_struct *work, void (*fn)(struct work_struct *))
484 1.1.24.1 pgoyette {
485 1.1 skrll
486 1.1 skrll work->work_owner = 0;
487 1.1.24.1 pgoyette work->func = fn;
488 1.1.24.1 pgoyette }
489 1.1.24.1 pgoyette
490 1.1.24.1 pgoyette /*
491 1.1.24.1 pgoyette * work_claimed(work, wq)
492 1.1.24.1 pgoyette *
493 1.1.24.1 pgoyette * True if work is currently claimed by a workqueue, meaning it is
494 1.1.24.1 pgoyette * either on the queue or scheduled in a callout. The workqueue
495 1.1.24.1 pgoyette * must be wq, and caller must hold wq's lock.
496 1.1 skrll */
497 1.1 skrll static bool
498 1.1.24.1 pgoyette work_claimed(struct work_struct *work, struct workqueue_struct *wq)
499 1.1.24.1 pgoyette {
500 1.1 skrll
501 1.1.24.1 pgoyette KASSERT(work_queue(work) == wq);
502 1.1 skrll KASSERT(mutex_owned(&wq->wq_lock));
503 1.1 skrll
504 1.1.24.1 pgoyette return work->work_owner & 1;
505 1.1.24.1 pgoyette }
506 1.1.24.1 pgoyette
507 1.1.24.1 pgoyette /*
508 1.1.24.1 pgoyette * work_queue(work)
509 1.1.24.1 pgoyette *
510 1.1.24.1 pgoyette * Return the last queue that work was queued on, or NULL if it
511 1.1.24.1 pgoyette * was never queued.
512 1.1 skrll */
513 1.1 skrll static struct workqueue_struct *
514 1.1.24.1 pgoyette work_queue(struct work_struct *work)
515 1.1 skrll {
516 1.1 skrll
517 1.1.24.1 pgoyette return (struct workqueue_struct *)(work->work_owner & ~(uintptr_t)1);
518 1.1.24.1 pgoyette }
519 1.1.24.1 pgoyette
520 1.1.24.1 pgoyette /*
521 1.1.24.1 pgoyette * acquire_work(work, wq)
522 1.1.24.1 pgoyette *
523 1.1.24.1 pgoyette * Try to claim work for wq. If work is already claimed, it must
524 1.1.24.1 pgoyette * be claimed by wq; return false. If work is not already
525 1.1.24.1 pgoyette * claimed, claim it, issue a memory barrier to match any prior
526 1.1.24.1 pgoyette * release_work, and return true.
527 1.1.24.1 pgoyette *
528 1.1.24.1 pgoyette * Caller must hold wq's lock.
529 1.1 skrll */
530 1.1.24.1 pgoyette static bool
531 1.1.24.1 pgoyette acquire_work(struct work_struct *work, struct workqueue_struct *wq)
532 1.1.24.1 pgoyette {
533 1.1.24.1 pgoyette uintptr_t owner0, owner;
534 1.1.24.1 pgoyette
535 1.1.24.1 pgoyette KASSERT(mutex_owned(&wq->wq_lock));
536 1.1.24.1 pgoyette KASSERT(((uintptr_t)wq & 1) == 0);
537 1.1.24.1 pgoyette
538 1.1.24.1 pgoyette owner = (uintptr_t)wq | 1;
539 1.1.24.1 pgoyette do {
540 1.1.24.1 pgoyette owner0 = work->work_owner;
541 1.1.24.1 pgoyette if (owner0 & 1) {
542 1.1.24.1 pgoyette KASSERT((owner0 & ~(uintptr_t)1) == (uintptr_t)wq);
543 1.1.24.1 pgoyette return false;
544 1.1.24.1 pgoyette }
545 1.1.24.1 pgoyette KASSERT(owner0 == (uintptr_t)NULL || owner0 == (uintptr_t)wq);
546 1.1.24.1 pgoyette } while (atomic_cas_uintptr(&work->work_owner, owner0, owner) !=
547 1.1.24.1 pgoyette owner0);
548 1.1.24.1 pgoyette
549 1.1.24.1 pgoyette KASSERT(work_queue(work) == wq);
550 1.1 skrll membar_enter();
551 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, acquire, work, wq);
552 1.1 skrll return true;
553 1.1.24.1 pgoyette }
554 1.1.24.1 pgoyette
555 1.1.24.1 pgoyette /*
556 1.1.24.1 pgoyette * release_work(work, wq)
557 1.1.24.1 pgoyette *
558 1.1.24.1 pgoyette * Issue a memory barrier to match any subsequent acquire_work and
559 1.1 skrll * dissociate work from wq.
560 1.1.24.1 pgoyette *
561 1.1.24.1 pgoyette * Caller must hold wq's lock and work must be associated with wq.
562 1.1 skrll */
563 1.1 skrll static void
564 1.1.24.1 pgoyette release_work(struct work_struct *work, struct workqueue_struct *wq)
565 1.1.24.1 pgoyette {
566 1.1.24.1 pgoyette
567 1.1.24.1 pgoyette KASSERT(work_queue(work) == wq);
568 1.1.24.1 pgoyette KASSERT(mutex_owned(&wq->wq_lock));
569 1.1.24.1 pgoyette
570 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, release, work, wq);
571 1.1.24.1 pgoyette membar_exit();
572 1.1.24.1 pgoyette
573 1.1.24.1 pgoyette /*
574 1.1.24.1 pgoyette * Non-interlocked r/m/w is safe here because nobody else can
575 1.1.24.1 pgoyette * write to this while the claimed bit is setand the workqueue
576 1.1 skrll * lock is held.
577 1.1 skrll */
578 1.1.24.1 pgoyette work->work_owner &= ~(uintptr_t)1;
579 1.1.24.1 pgoyette }
580 1.1.24.1 pgoyette
581 1.1.24.1 pgoyette /*
582 1.1.24.1 pgoyette * schedule_work(work)
583 1.1.24.1 pgoyette *
584 1.1.24.1 pgoyette * If work is not already queued on system_wq, queue it to be run
585 1.1.24.1 pgoyette * by system_wq's worker thread when it next can. True if it was
586 1.1.24.1 pgoyette * newly queued, false if it was already queued. If the work was
587 1.1.24.1 pgoyette * already running, queue it to run again.
588 1.1.24.1 pgoyette *
589 1.1 skrll * Caller must ensure work is not queued to run on a different
590 1.1 skrll * workqueue.
591 1.1 skrll */
592 1.1.24.1 pgoyette bool
593 1.1 skrll schedule_work(struct work_struct *work)
594 1.1 skrll {
595 1.1 skrll
596 1.1.24.1 pgoyette return queue_work(system_wq, work);
597 1.1.24.1 pgoyette }
598 1.1.24.1 pgoyette
599 1.1.24.1 pgoyette /*
600 1.1.24.1 pgoyette * queue_work(wq, work)
601 1.1.24.1 pgoyette *
602 1.1.24.1 pgoyette * If work is not already queued on wq, queue it to be run by wq's
603 1.1.24.1 pgoyette * worker thread when it next can. True if it was newly queued,
604 1.1.24.1 pgoyette * false if it was already queued. If the work was already
605 1.1.24.1 pgoyette * running, queue it to run again.
606 1.1.24.1 pgoyette *
607 1.1 skrll * Caller must ensure work is not queued to run on a different
608 1.1 skrll * workqueue.
609 1.1 skrll */
610 1.1 skrll bool
611 1.1 skrll queue_work(struct workqueue_struct *wq, struct work_struct *work)
612 1.1 skrll {
613 1.1 skrll bool newly_queued;
614 1.1.24.1 pgoyette
615 1.1.24.1 pgoyette KASSERT(wq != NULL);
616 1.1.24.1 pgoyette
617 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
618 1.1.24.1 pgoyette if (__predict_true(acquire_work(work, wq))) {
619 1.1.24.1 pgoyette /*
620 1.1.24.1 pgoyette * It wasn't on any workqueue at all. Put it on this
621 1.1.24.1 pgoyette * one, and signal the worker thread that there is work
622 1.1.24.1 pgoyette * to do.
623 1.1.24.1 pgoyette */
624 1.1 skrll TAILQ_INSERT_TAIL(&wq->wq_queue, work, work_entry);
625 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
626 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue, work, wq);
627 1.1.24.1 pgoyette newly_queued = true;
628 1.1.24.1 pgoyette } else {
629 1.1.24.1 pgoyette /*
630 1.1 skrll * It was already on this workqueue. Nothing to do
631 1.1 skrll * since it is already queued.
632 1.1.24.1 pgoyette */
633 1.1 skrll newly_queued = false;
634 1.1 skrll }
635 1.1 skrll mutex_exit(&wq->wq_lock);
636 1.1 skrll
637 1.1.24.1 pgoyette return newly_queued;
638 1.1.24.1 pgoyette }
639 1.1.24.1 pgoyette
640 1.1.24.1 pgoyette /*
641 1.1.24.1 pgoyette * cancel_work(work)
642 1.1.24.1 pgoyette *
643 1.1.24.1 pgoyette * If work was queued, remove it from the queue and return true.
644 1.1 skrll * If work was not queued, return false. Work may still be
645 1.1.24.1 pgoyette * running when this returns.
646 1.1 skrll */
647 1.1.24.1 pgoyette bool
648 1.1 skrll cancel_work(struct work_struct *work)
649 1.1 skrll {
650 1.1.24.1 pgoyette struct workqueue_struct *wq;
651 1.1.24.1 pgoyette bool cancelled_p = false;
652 1.1.24.1 pgoyette
653 1.1 skrll /* If there's no workqueue, nothing to cancel. */
654 1.1.24.1 pgoyette if ((wq = work_queue(work)) == NULL)
655 1.1.24.1 pgoyette goto out;
656 1.1.24.1 pgoyette
657 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
658 1.1.24.1 pgoyette if (__predict_false(work_queue(work) != wq)) {
659 1.1.24.1 pgoyette /*
660 1.1.24.1 pgoyette * It has finished execution or been cancelled by
661 1.1.24.1 pgoyette * another thread, and has been moved off the
662 1.1.24.1 pgoyette * workqueue, so it's too to cancel.
663 1.1.24.1 pgoyette */
664 1.1.24.1 pgoyette cancelled_p = false;
665 1.1.24.1 pgoyette } else {
666 1.1.24.1 pgoyette /* Check whether it's on the queue. */
667 1.1.24.1 pgoyette if (work_claimed(work, wq)) {
668 1.1.24.1 pgoyette /*
669 1.1.24.1 pgoyette * It is still on the queue. Take it off the
670 1.1.24.1 pgoyette * queue and report successful cancellation.
671 1.1.24.1 pgoyette */
672 1.1.24.1 pgoyette TAILQ_REMOVE(&wq->wq_queue, work, work_entry);
673 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, work, wq);
674 1.1.24.1 pgoyette release_work(work, wq);
675 1.1.24.1 pgoyette /* Can't dereference work after this point. */
676 1.1.24.1 pgoyette cancelled_p = true;
677 1.1.24.1 pgoyette } else {
678 1.1.24.1 pgoyette /* Not on the queue. Couldn't cancel it. */
679 1.1.24.1 pgoyette cancelled_p = false;
680 1.1 skrll }
681 1.1.24.1 pgoyette }
682 1.1.24.1 pgoyette mutex_exit(&wq->wq_lock);
683 1.1 skrll
684 1.1.24.1 pgoyette out: return cancelled_p;
685 1.1.24.1 pgoyette }
686 1.1.24.1 pgoyette
687 1.1.24.1 pgoyette /*
688 1.1.24.1 pgoyette * cancel_work_sync(work)
689 1.1.24.1 pgoyette *
690 1.1.24.1 pgoyette * If work was queued, remove it from the queue and return true.
691 1.1.24.1 pgoyette * If work was not queued, return false. Either way, if work is
692 1.1.24.1 pgoyette * currently running, wait for it to complete.
693 1.1.24.1 pgoyette *
694 1.1.24.1 pgoyette * May sleep.
695 1.1.24.1 pgoyette */
696 1.1.24.1 pgoyette bool
697 1.1.24.1 pgoyette cancel_work_sync(struct work_struct *work)
698 1.1 skrll {
699 1.1.24.1 pgoyette struct workqueue_struct *wq;
700 1.1.24.1 pgoyette bool cancelled_p = false;
701 1.1.24.1 pgoyette
702 1.1 skrll /* If there's no workqueue, nothing to cancel. */
703 1.1.24.1 pgoyette if ((wq = work_queue(work)) == NULL)
704 1.1.24.1 pgoyette goto out;
705 1.1.24.1 pgoyette
706 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
707 1.1.24.1 pgoyette if (__predict_false(work_queue(work) != wq)) {
708 1.1.24.1 pgoyette /*
709 1.1.24.1 pgoyette * It has finished execution or been cancelled by
710 1.1.24.1 pgoyette * another thread, and has been moved off the
711 1.1.24.1 pgoyette * workqueue, so it's too late to cancel.
712 1.1.24.1 pgoyette */
713 1.1.24.1 pgoyette cancelled_p = false;
714 1.1.24.1 pgoyette } else {
715 1.1.24.1 pgoyette /* Check whether it's on the queue. */
716 1.1.24.1 pgoyette if (work_claimed(work, wq)) {
717 1.1.24.1 pgoyette /*
718 1.1.24.1 pgoyette * It is still on the queue. Take it off the
719 1.1.24.1 pgoyette * queue and report successful cancellation.
720 1.1.24.1 pgoyette */
721 1.1.24.1 pgoyette TAILQ_REMOVE(&wq->wq_queue, work, work_entry);
722 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, work, wq);
723 1.1.24.1 pgoyette release_work(work, wq);
724 1.1.24.1 pgoyette /* Can't dereference work after this point. */
725 1.1.24.1 pgoyette cancelled_p = true;
726 1.1.24.1 pgoyette } else {
727 1.1.24.1 pgoyette /* Not on the queue. Couldn't cancel it. */
728 1.1.24.1 pgoyette cancelled_p = false;
729 1.1.24.1 pgoyette }
730 1.1 skrll /* If it's still running, wait for it to complete. */
731 1.1.24.1 pgoyette if (wq->wq_current_work == work)
732 1.1 skrll wait_for_current_work(work, wq);
733 1.1.24.1 pgoyette }
734 1.1 skrll mutex_exit(&wq->wq_lock);
735 1.1 skrll
736 1.1.24.1 pgoyette out: return cancelled_p;
737 1.1.24.1 pgoyette }
738 1.1.24.1 pgoyette
739 1.1.24.1 pgoyette /*
740 1.1.24.1 pgoyette * wait_for_current_work(work, wq)
741 1.1.24.1 pgoyette *
742 1.1.24.1 pgoyette * wq must be currently executing work. Wait for it to finish.
743 1.1 skrll *
744 1.1.24.1 pgoyette * Does not dereference work.
745 1.1 skrll */
746 1.1.24.1 pgoyette static void
747 1.1 skrll wait_for_current_work(struct work_struct *work, struct workqueue_struct *wq)
748 1.1.24.1 pgoyette {
749 1.1.24.1 pgoyette uint64_t gen;
750 1.1 skrll
751 1.1.24.1 pgoyette KASSERT(mutex_owned(&wq->wq_lock));
752 1.1.24.1 pgoyette KASSERT(wq->wq_current_work == work);
753 1.1.24.1 pgoyette
754 1.1 skrll /* Wait only one generation in case it gets requeued quickly. */
755 1.1 skrll SDT_PROBE2(sdt, linux, work, wait__start, work, wq);
756 1.1.24.1 pgoyette gen = wq->wq_gen;
757 1.1.24.1 pgoyette do {
758 1.1 skrll cv_wait(&wq->wq_cv, &wq->wq_lock);
759 1.1.24.1 pgoyette } while (wq->wq_current_work == work && wq->wq_gen == gen);
760 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, wait__done, work, wq);
761 1.1.24.1 pgoyette }
762 1.1.24.1 pgoyette
763 1.1 skrll /*
765 1.1.24.1 pgoyette * Delayed work
766 1.1.24.1 pgoyette */
767 1.1.24.1 pgoyette
768 1.1.24.1 pgoyette /*
769 1.1.24.1 pgoyette * INIT_DELAYED_WORK(dw, fn)
770 1.1.24.1 pgoyette *
771 1.1.24.1 pgoyette * Initialize dw for use with a workqueue to call fn in a worker
772 1.1.24.1 pgoyette * thread after a delay. There is no corresponding destruction
773 1.1 skrll * operation.
774 1.1 skrll */
775 1.1.24.1 pgoyette void
776 1.1.24.1 pgoyette INIT_DELAYED_WORK(struct delayed_work *dw, void (*fn)(struct work_struct *))
777 1.1.24.1 pgoyette {
778 1.1 skrll
779 1.1.24.1 pgoyette INIT_WORK(&dw->work, fn);
780 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_IDLE;
781 1.1.24.1 pgoyette dw->dw_resched = -1;
782 1.1.24.1 pgoyette
783 1.1.24.1 pgoyette /*
784 1.1.24.1 pgoyette * Defer callout_init until we are going to schedule the
785 1.1 skrll * callout, which can then callout_destroy it, because
786 1.1 skrll * otherwise since there's no DESTROY_DELAYED_WORK or anything
787 1.1.24.1 pgoyette * we have no opportunity to call callout_destroy.
788 1.1.24.1 pgoyette */
789 1.1.24.1 pgoyette }
790 1.1.24.1 pgoyette
791 1.1.24.1 pgoyette /*
792 1.1.24.1 pgoyette * schedule_delayed_work(dw, ticks)
793 1.1.24.1 pgoyette *
794 1.1.24.1 pgoyette * If it is not currently scheduled, schedule dw to run after
795 1.1.24.1 pgoyette * ticks on system_wq. If currently executing and not already
796 1.1.24.1 pgoyette * rescheduled, reschedule it. True if it was newly scheduled,
797 1.1.24.1 pgoyette * false if it was already scheduled.
798 1.1.24.1 pgoyette *
799 1.1.24.1 pgoyette * If ticks == 0, queue it to run as soon as the worker can,
800 1.1 skrll * without waiting for the next callout tick to run.
801 1.1 skrll */
802 1.1.24.1 pgoyette bool
803 1.1.24.1 pgoyette schedule_delayed_work(struct delayed_work *dw, unsigned long ticks)
804 1.1 skrll {
805 1.1.24.1 pgoyette
806 1.1.24.1 pgoyette return queue_delayed_work(system_wq, dw, ticks);
807 1.1.24.1 pgoyette }
808 1.1.24.1 pgoyette
809 1.1.24.1 pgoyette /*
810 1.1.24.1 pgoyette * dw_callout_init(wq, dw)
811 1.1.24.1 pgoyette *
812 1.1.24.1 pgoyette * Initialize the callout of dw and transition to
813 1.1.24.1 pgoyette * DELAYED_WORK_SCHEDULED. Caller must use callout_schedule.
814 1.1 skrll */
815 1.1.24.1 pgoyette static void
816 1.1.24.1 pgoyette dw_callout_init(struct workqueue_struct *wq, struct delayed_work *dw)
817 1.1.24.1 pgoyette {
818 1.1 skrll
819 1.1.24.1 pgoyette KASSERT(mutex_owned(&wq->wq_lock));
820 1.1.24.1 pgoyette KASSERT(work_queue(&dw->work) == wq);
821 1.1.24.1 pgoyette KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
822 1.1.24.1 pgoyette
823 1.1.24.1 pgoyette callout_init(&dw->dw_callout, CALLOUT_MPSAFE);
824 1.1 skrll callout_setfunc(&dw->dw_callout, &linux_workqueue_timeout, dw);
825 1.1.24.1 pgoyette TAILQ_INSERT_HEAD(&wq->wq_delayed, dw, dw_entry);
826 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_SCHEDULED;
827 1.1.24.1 pgoyette }
828 1.1.24.1 pgoyette
829 1.1.24.1 pgoyette /*
830 1.1.24.1 pgoyette * dw_callout_destroy(wq, dw)
831 1.1.24.1 pgoyette *
832 1.1.24.1 pgoyette * Destroy the callout of dw and transition to DELAYED_WORK_IDLE.
833 1.1 skrll */
834 1.1.24.1 pgoyette static void
835 1.1.24.1 pgoyette dw_callout_destroy(struct workqueue_struct *wq, struct delayed_work *dw)
836 1.1.24.1 pgoyette {
837 1.1.24.1 pgoyette
838 1.1.24.1 pgoyette KASSERT(mutex_owned(&wq->wq_lock));
839 1.1 skrll KASSERT(work_queue(&dw->work) == wq);
840 1.1.24.1 pgoyette KASSERT(dw->dw_state == DELAYED_WORK_SCHEDULED ||
841 1.1.24.1 pgoyette dw->dw_state == DELAYED_WORK_RESCHEDULED ||
842 1.1.24.1 pgoyette dw->dw_state == DELAYED_WORK_CANCELLED);
843 1.1.24.1 pgoyette
844 1.1 skrll TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
845 1.1.24.1 pgoyette callout_destroy(&dw->dw_callout);
846 1.1 skrll dw->dw_resched = -1;
847 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_IDLE;
848 1.1.24.1 pgoyette }
849 1.1.24.1 pgoyette
850 1.1.24.1 pgoyette /*
851 1.1.24.1 pgoyette * cancel_delayed_work_done(wq, dw)
852 1.1 skrll *
853 1.1.24.1 pgoyette * Complete cancellation of a delayed work: transition from
854 1.1.24.1 pgoyette * DELAYED_WORK_CANCELLED to DELAYED_WORK_IDLE and off the
855 1.1 skrll * workqueue. Caller must not dereference dw after this returns.
856 1.1 skrll */
857 1.1.24.1 pgoyette static void
858 1.1.24.1 pgoyette cancel_delayed_work_done(struct workqueue_struct *wq, struct delayed_work *dw)
859 1.1.24.1 pgoyette {
860 1.1.24.1 pgoyette
861 1.1.24.1 pgoyette KASSERT(mutex_owned(&wq->wq_lock));
862 1.1.24.1 pgoyette KASSERT(work_queue(&dw->work) == wq);
863 1.1.24.1 pgoyette KASSERT(dw->dw_state == DELAYED_WORK_CANCELLED);
864 1.1 skrll
865 1.1 skrll dw_callout_destroy(wq, dw);
866 1.1.24.1 pgoyette release_work(&dw->work, wq);
867 1.1.24.1 pgoyette /* Can't dereference dw after this point. */
868 1.1.24.1 pgoyette }
869 1.1.24.1 pgoyette
870 1.1.24.1 pgoyette /*
871 1.1.24.1 pgoyette * queue_delayed_work(wq, dw, ticks)
872 1.1.24.1 pgoyette *
873 1.1.24.1 pgoyette * If it is not currently scheduled, schedule dw to run after
874 1.1.24.1 pgoyette * ticks on wq. If currently queued, remove it from the queue
875 1.1.24.1 pgoyette * first.
876 1.1 skrll *
877 1.1 skrll * If ticks == 0, queue it to run as soon as the worker can,
878 1.1 skrll * without waiting for the next callout tick to run.
879 1.1 skrll */
880 1.1 skrll bool
881 1.1 skrll queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
882 1.1.24.1 pgoyette unsigned long ticks)
883 1.1.24.1 pgoyette {
884 1.1.24.1 pgoyette bool newly_queued;
885 1.1.24.1 pgoyette
886 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
887 1.1.24.1 pgoyette if (__predict_true(acquire_work(&dw->work, wq))) {
888 1.1.24.1 pgoyette /*
889 1.1 skrll * It wasn't on any workqueue at all. Schedule it to
890 1.1.24.1 pgoyette * run on this one.
891 1.1.24.1 pgoyette */
892 1.1.24.1 pgoyette KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
893 1.1.24.1 pgoyette if (ticks == 0) {
894 1.1 skrll TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work,
895 1.1.24.1 pgoyette work_entry);
896 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
897 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue, &dw->work, wq);
898 1.1.24.1 pgoyette } else {
899 1.1.24.1 pgoyette /*
900 1.1.24.1 pgoyette * Initialize a callout and schedule to run
901 1.1.24.1 pgoyette * after a delay.
902 1.1 skrll */
903 1.1 skrll dw_callout_init(wq, dw);
904 1.1.24.1 pgoyette callout_schedule(&dw->dw_callout, MIN(INT_MAX, ticks));
905 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule, dw, wq, ticks);
906 1.1.24.1 pgoyette }
907 1.1.24.1 pgoyette newly_queued = true;
908 1.1.24.1 pgoyette } else {
909 1.1.24.1 pgoyette /* It was already on this workqueue. */
910 1.1.24.1 pgoyette switch (dw->dw_state) {
911 1.1.24.1 pgoyette case DELAYED_WORK_IDLE:
912 1.1.24.1 pgoyette case DELAYED_WORK_SCHEDULED:
913 1.1.24.1 pgoyette case DELAYED_WORK_RESCHEDULED:
914 1.1.24.1 pgoyette /* On the queue or already scheduled. Leave it. */
915 1.1.24.1 pgoyette newly_queued = false;
916 1.1.24.1 pgoyette break;
917 1.1.24.1 pgoyette case DELAYED_WORK_CANCELLED:
918 1.1.24.1 pgoyette /*
919 1.1.24.1 pgoyette * Scheduled and the callout began, but it was
920 1.1.24.1 pgoyette * cancelled. Reschedule it.
921 1.1.24.1 pgoyette */
922 1.1.24.1 pgoyette if (ticks == 0) {
923 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_SCHEDULED;
924 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
925 1.1.24.1 pgoyette &dw->work, wq);
926 1.1.24.1 pgoyette } else {
927 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_RESCHEDULED;
928 1.1.24.1 pgoyette dw->dw_resched = MIN(INT_MAX, ticks);
929 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule,
930 1.1.24.1 pgoyette dw, wq, ticks);
931 1.1.24.1 pgoyette }
932 1.1.24.1 pgoyette newly_queued = true;
933 1.1.24.1 pgoyette break;
934 1.1 skrll default:
935 1.1.24.1 pgoyette panic("invalid delayed work state: %d",
936 1.1 skrll dw->dw_state);
937 1.1 skrll }
938 1.1 skrll }
939 1.1 skrll mutex_exit(&wq->wq_lock);
940 1.1.24.1 pgoyette
941 1.1.24.1 pgoyette return newly_queued;
942 1.1.24.1 pgoyette }
943 1.1.24.1 pgoyette
944 1.1.24.1 pgoyette /*
945 1.1.24.1 pgoyette * mod_delayed_work(wq, dw, ticks)
946 1.1.24.1 pgoyette *
947 1.1.24.1 pgoyette * Schedule dw to run after ticks. If scheduled or queued,
948 1.1.24.1 pgoyette * reschedule. If ticks == 0, run without delay.
949 1.1 skrll *
950 1.1 skrll * True if it modified the timer of an already scheduled work,
951 1.1 skrll * false if it newly scheduled the work.
952 1.1 skrll */
953 1.1 skrll bool
954 1.1 skrll mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
955 1.1.24.1 pgoyette unsigned long ticks)
956 1.1.24.1 pgoyette {
957 1.1.24.1 pgoyette bool timer_modified;
958 1.1.24.1 pgoyette
959 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
960 1.1.24.1 pgoyette if (acquire_work(&dw->work, wq)) {
961 1.1.24.1 pgoyette /*
962 1.1 skrll * It wasn't on any workqueue at all. Schedule it to
963 1.1.24.1 pgoyette * run on this one.
964 1.1.24.1 pgoyette */
965 1.1.24.1 pgoyette KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
966 1.1.24.1 pgoyette if (ticks == 0) {
967 1.1.24.1 pgoyette /*
968 1.1.24.1 pgoyette * Run immediately: put it on the queue and
969 1.1.24.1 pgoyette * signal the worker thread.
970 1.1.24.1 pgoyette */
971 1.1 skrll TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work,
972 1.1.24.1 pgoyette work_entry);
973 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
974 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue, &dw->work, wq);
975 1.1.24.1 pgoyette } else {
976 1.1.24.1 pgoyette /*
977 1.1.24.1 pgoyette * Initialize a callout and schedule to run
978 1.1.24.1 pgoyette * after a delay.
979 1.1 skrll */
980 1.1 skrll dw_callout_init(wq, dw);
981 1.1.24.1 pgoyette callout_schedule(&dw->dw_callout, MIN(INT_MAX, ticks));
982 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule, dw, wq, ticks);
983 1.1.24.1 pgoyette }
984 1.1.24.1 pgoyette timer_modified = false;
985 1.1.24.1 pgoyette } else {
986 1.1.24.1 pgoyette /* It was already on this workqueue. */
987 1.1.24.1 pgoyette switch (dw->dw_state) {
988 1.1.24.1 pgoyette case DELAYED_WORK_IDLE:
989 1.1.24.1 pgoyette /* On the queue. */
990 1.1.24.1 pgoyette if (ticks == 0) {
991 1.1.24.1 pgoyette /* Leave it be. */
992 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
993 1.1.24.1 pgoyette &dw->work, wq);
994 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
995 1.1.24.1 pgoyette &dw->work, wq);
996 1.1.24.1 pgoyette } else {
997 1.1.24.1 pgoyette /* Remove from the queue and schedule. */
998 1.1.24.1 pgoyette TAILQ_REMOVE(&wq->wq_dqueue, &dw->work,
999 1.1.24.1 pgoyette work_entry);
1000 1.1.24.1 pgoyette dw_callout_init(wq, dw);
1001 1.1.24.1 pgoyette callout_schedule(&dw->dw_callout,
1002 1.1.24.1 pgoyette MIN(INT_MAX, ticks));
1003 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1004 1.1.24.1 pgoyette &dw->work, wq);
1005 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule,
1006 1.1.24.1 pgoyette dw, wq, ticks);
1007 1.1.24.1 pgoyette }
1008 1.1.24.1 pgoyette timer_modified = true;
1009 1.1.24.1 pgoyette break;
1010 1.1.24.1 pgoyette case DELAYED_WORK_SCHEDULED:
1011 1.1.24.1 pgoyette /*
1012 1.1.24.1 pgoyette * It is scheduled to run after a delay. Try
1013 1.1.24.1 pgoyette * to stop it and reschedule it; if we can't,
1014 1.1.24.1 pgoyette * either reschedule it or cancel it to put it
1015 1.1.24.1 pgoyette * on the queue, and inform the callout.
1016 1.1.24.1 pgoyette */
1017 1.1.24.1 pgoyette if (callout_stop(&dw->dw_callout)) {
1018 1.1.24.1 pgoyette /* Can't stop, callout has begun. */
1019 1.1.24.1 pgoyette if (ticks == 0) {
1020 1.1.24.1 pgoyette /*
1021 1.1.24.1 pgoyette * We don't actually need to do
1022 1.1.24.1 pgoyette * anything. The callout will
1023 1.1.24.1 pgoyette * queue it as soon as it gets
1024 1.1.24.1 pgoyette * the lock.
1025 1.1.24.1 pgoyette */
1026 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1027 1.1.24.1 pgoyette &dw->work, wq);
1028 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
1029 1.1.24.1 pgoyette &dw->work, wq);
1030 1.1.24.1 pgoyette } else {
1031 1.1.24.1 pgoyette /* Ask the callout to reschedule. */
1032 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_RESCHEDULED;
1033 1.1.24.1 pgoyette dw->dw_resched = MIN(INT_MAX, ticks);
1034 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1035 1.1.24.1 pgoyette &dw->work, wq);
1036 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule,
1037 1.1.24.1 pgoyette dw, wq, ticks);
1038 1.1.24.1 pgoyette }
1039 1.1.24.1 pgoyette } else {
1040 1.1.24.1 pgoyette /* We stopped the callout before it began. */
1041 1.1.24.1 pgoyette if (ticks == 0) {
1042 1.1.24.1 pgoyette /*
1043 1.1.24.1 pgoyette * Run immediately: destroy the
1044 1.1.24.1 pgoyette * callout, put it on the
1045 1.1.24.1 pgoyette * queue, and signal the worker
1046 1.1.24.1 pgoyette * thread.
1047 1.1.24.1 pgoyette */
1048 1.1.24.1 pgoyette dw_callout_destroy(wq, dw);
1049 1.1.24.1 pgoyette TAILQ_INSERT_TAIL(&wq->wq_dqueue,
1050 1.1.24.1 pgoyette &dw->work, work_entry);
1051 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
1052 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1053 1.1.24.1 pgoyette &dw->work, wq);
1054 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
1055 1.1.24.1 pgoyette &dw->work, wq);
1056 1.1.24.1 pgoyette } else {
1057 1.1.24.1 pgoyette /*
1058 1.1.24.1 pgoyette * Reschedule the callout. No
1059 1.1.24.1 pgoyette * state change.
1060 1.1.24.1 pgoyette */
1061 1.1.24.1 pgoyette callout_schedule(&dw->dw_callout,
1062 1.1.24.1 pgoyette MIN(INT_MAX, ticks));
1063 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1064 1.1.24.1 pgoyette &dw->work, wq);
1065 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule,
1066 1.1.24.1 pgoyette dw, wq, ticks);
1067 1.1.24.1 pgoyette }
1068 1.1.24.1 pgoyette }
1069 1.1.24.1 pgoyette timer_modified = true;
1070 1.1.24.1 pgoyette break;
1071 1.1.24.1 pgoyette case DELAYED_WORK_RESCHEDULED:
1072 1.1.24.1 pgoyette /*
1073 1.1.24.1 pgoyette * Someone rescheduled it after the callout
1074 1.1.24.1 pgoyette * started but before the poor thing even had a
1075 1.1.24.1 pgoyette * chance to acquire the lock.
1076 1.1.24.1 pgoyette */
1077 1.1.24.1 pgoyette if (ticks == 0) {
1078 1.1.24.1 pgoyette /*
1079 1.1.24.1 pgoyette * We can just switch back to
1080 1.1.24.1 pgoyette * DELAYED_WORK_SCHEDULED so that the
1081 1.1.24.1 pgoyette * callout will queue the work as soon
1082 1.1.24.1 pgoyette * as it gets the lock.
1083 1.1.24.1 pgoyette */
1084 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_SCHEDULED;
1085 1.1.24.1 pgoyette dw->dw_resched = -1;
1086 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1087 1.1.24.1 pgoyette &dw->work, wq);
1088 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
1089 1.1.24.1 pgoyette &dw->work, wq);
1090 1.1.24.1 pgoyette } else {
1091 1.1.24.1 pgoyette /* Change the rescheduled time. */
1092 1.1.24.1 pgoyette dw->dw_resched = ticks;
1093 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1094 1.1.24.1 pgoyette &dw->work, wq);
1095 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule,
1096 1.1.24.1 pgoyette dw, wq, ticks);
1097 1.1.24.1 pgoyette }
1098 1.1.24.1 pgoyette timer_modified = true;
1099 1.1.24.1 pgoyette break;
1100 1.1.24.1 pgoyette case DELAYED_WORK_CANCELLED:
1101 1.1.24.1 pgoyette /*
1102 1.1.24.1 pgoyette * Someone cancelled it after the callout
1103 1.1.24.1 pgoyette * started but before the poor thing even had a
1104 1.1.24.1 pgoyette * chance to acquire the lock.
1105 1.1.24.1 pgoyette */
1106 1.1.24.1 pgoyette if (ticks == 0) {
1107 1.1.24.1 pgoyette /*
1108 1.1.24.1 pgoyette * We can just switch back to
1109 1.1.24.1 pgoyette * DELAYED_WORK_SCHEDULED so that the
1110 1.1.24.1 pgoyette * callout will queue the work as soon
1111 1.1.24.1 pgoyette * as it gets the lock.
1112 1.1.24.1 pgoyette */
1113 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_SCHEDULED;
1114 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
1115 1.1.24.1 pgoyette &dw->work, wq);
1116 1.1.24.1 pgoyette } else {
1117 1.1.24.1 pgoyette /* Ask it to reschedule. */
1118 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_RESCHEDULED;
1119 1.1.24.1 pgoyette dw->dw_resched = MIN(INT_MAX, ticks);
1120 1.1.24.1 pgoyette SDT_PROBE3(sdt, linux, work, schedule,
1121 1.1.24.1 pgoyette dw, wq, ticks);
1122 1.1.24.1 pgoyette }
1123 1.1.24.1 pgoyette timer_modified = false;
1124 1.1 skrll break;
1125 1.1.24.1 pgoyette default:
1126 1.1 skrll panic("invalid delayed work state: %d", dw->dw_state);
1127 1.1 skrll }
1128 1.1 skrll }
1129 1.1 skrll mutex_exit(&wq->wq_lock);
1130 1.1.24.1 pgoyette
1131 1.1.24.1 pgoyette return timer_modified;
1132 1.1.24.1 pgoyette }
1133 1.1.24.1 pgoyette
1134 1.1.24.1 pgoyette /*
1135 1.1.24.1 pgoyette * cancel_delayed_work(dw)
1136 1.1.24.1 pgoyette *
1137 1.1.24.1 pgoyette * If work was scheduled or queued, remove it from the schedule or
1138 1.1.24.1 pgoyette * queue and return true. If work was not scheduled or queued,
1139 1.1.24.1 pgoyette * return false. Note that work may already be running; if it
1140 1.1 skrll * hasn't been rescheduled or requeued, then cancel_delayed_work
1141 1.1 skrll * will return false, and either way, cancel_delayed_work will NOT
1142 1.1 skrll * wait for the work to complete.
1143 1.1.24.1 pgoyette */
1144 1.1.24.1 pgoyette bool
1145 1.1 skrll cancel_delayed_work(struct delayed_work *dw)
1146 1.1.24.1 pgoyette {
1147 1.1.24.1 pgoyette struct workqueue_struct *wq;
1148 1.1.24.1 pgoyette bool cancelled_p;
1149 1.1 skrll
1150 1.1.24.1 pgoyette /* If there's no workqueue, nothing to cancel. */
1151 1.1.24.1 pgoyette if ((wq = work_queue(&dw->work)) == NULL)
1152 1.1.24.1 pgoyette return false;
1153 1.1.24.1 pgoyette
1154 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
1155 1.1.24.1 pgoyette if (__predict_false(work_queue(&dw->work) != wq)) {
1156 1.1.24.1 pgoyette cancelled_p = false;
1157 1.1.24.1 pgoyette } else {
1158 1.1.24.1 pgoyette switch (dw->dw_state) {
1159 1.1.24.1 pgoyette case DELAYED_WORK_IDLE:
1160 1.1.24.1 pgoyette /*
1161 1.1.24.1 pgoyette * It is either on the queue or already running
1162 1.1.24.1 pgoyette * or both.
1163 1.1.24.1 pgoyette */
1164 1.1.24.1 pgoyette if (work_claimed(&dw->work, wq)) {
1165 1.1.24.1 pgoyette /* On the queue. Remove and release. */
1166 1.1.24.1 pgoyette TAILQ_REMOVE(&wq->wq_dqueue, &dw->work,
1167 1.1.24.1 pgoyette work_entry);
1168 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1169 1.1.24.1 pgoyette &dw->work, wq);
1170 1.1.24.1 pgoyette release_work(&dw->work, wq);
1171 1.1.24.1 pgoyette /* Can't dereference dw after this point. */
1172 1.1.24.1 pgoyette cancelled_p = true;
1173 1.1.24.1 pgoyette } else {
1174 1.1.24.1 pgoyette /* Not on the queue, so didn't cancel. */
1175 1.1.24.1 pgoyette cancelled_p = false;
1176 1.1.24.1 pgoyette }
1177 1.1.24.1 pgoyette break;
1178 1.1.24.1 pgoyette case DELAYED_WORK_SCHEDULED:
1179 1.1.24.1 pgoyette /*
1180 1.1.24.1 pgoyette * If it is scheduled, mark it cancelled and
1181 1.1.24.1 pgoyette * try to stop the callout before it starts.
1182 1.1.24.1 pgoyette *
1183 1.1.24.1 pgoyette * If it's too late and the callout has already
1184 1.1.24.1 pgoyette * begun to execute, tough.
1185 1.1.24.1 pgoyette *
1186 1.1.24.1 pgoyette * If we stopped the callout before it started,
1187 1.1.24.1 pgoyette * however, then destroy the callout and
1188 1.1.24.1 pgoyette * dissociate it from the workqueue ourselves.
1189 1.1.24.1 pgoyette */
1190 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_CANCELLED;
1191 1.1.24.1 pgoyette cancelled_p = true;
1192 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq);
1193 1.1.24.1 pgoyette if (!callout_stop(&dw->dw_callout))
1194 1.1.24.1 pgoyette cancel_delayed_work_done(wq, dw);
1195 1.1.24.1 pgoyette break;
1196 1.1.24.1 pgoyette case DELAYED_WORK_RESCHEDULED:
1197 1.1.24.1 pgoyette /*
1198 1.1.24.1 pgoyette * If it is being rescheduled, the callout has
1199 1.1.24.1 pgoyette * already fired. We must ask it to cancel.
1200 1.1.24.1 pgoyette */
1201 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_CANCELLED;
1202 1.1.24.1 pgoyette dw->dw_resched = -1;
1203 1.1.24.1 pgoyette cancelled_p = true;
1204 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq);
1205 1.1.24.1 pgoyette break;
1206 1.1.24.1 pgoyette case DELAYED_WORK_CANCELLED:
1207 1.1.24.1 pgoyette /*
1208 1.1.24.1 pgoyette * If it is being cancelled, the callout has
1209 1.1.24.1 pgoyette * already fired. There is nothing more for us
1210 1.1.24.1 pgoyette * to do. Someone else claims credit for
1211 1.1.24.1 pgoyette * cancelling it.
1212 1.1.24.1 pgoyette */
1213 1.1.24.1 pgoyette cancelled_p = false;
1214 1.1.24.1 pgoyette break;
1215 1.1 skrll default:
1216 1.1.24.1 pgoyette panic("invalid delayed work state: %d",
1217 1.1 skrll dw->dw_state);
1218 1.1 skrll }
1219 1.1 skrll }
1220 1.1 skrll mutex_exit(&wq->wq_lock);
1221 1.1.24.1 pgoyette
1222 1.1.24.1 pgoyette return cancelled_p;
1223 1.1.24.1 pgoyette }
1224 1.1.24.1 pgoyette
1225 1.1.24.1 pgoyette /*
1226 1.1.24.1 pgoyette * cancel_delayed_work_sync(dw)
1227 1.1.24.1 pgoyette *
1228 1.1.24.1 pgoyette * If work was scheduled or queued, remove it from the schedule or
1229 1.1.24.1 pgoyette * queue and return true. If work was not scheduled or queued,
1230 1.1 skrll * return false. Note that work may already be running; if it
1231 1.1 skrll * hasn't been rescheduled or requeued, then cancel_delayed_work
1232 1.1 skrll * will return false; either way, wait for it to complete.
1233 1.1.24.1 pgoyette */
1234 1.1.24.1 pgoyette bool
1235 1.1 skrll cancel_delayed_work_sync(struct delayed_work *dw)
1236 1.1.24.1 pgoyette {
1237 1.1.24.1 pgoyette struct workqueue_struct *wq;
1238 1.1.24.1 pgoyette bool cancelled_p;
1239 1.1 skrll
1240 1.1.24.1 pgoyette /* If there's no workqueue, nothing to cancel. */
1241 1.1.24.1 pgoyette if ((wq = work_queue(&dw->work)) == NULL)
1242 1.1.24.1 pgoyette return false;
1243 1.1.24.1 pgoyette
1244 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
1245 1.1.24.1 pgoyette if (__predict_false(work_queue(&dw->work) != wq)) {
1246 1.1.24.1 pgoyette cancelled_p = false;
1247 1.1.24.1 pgoyette } else {
1248 1.1.24.1 pgoyette switch (dw->dw_state) {
1249 1.1.24.1 pgoyette case DELAYED_WORK_IDLE:
1250 1.1.24.1 pgoyette /*
1251 1.1.24.1 pgoyette * It is either on the queue or already running
1252 1.1.24.1 pgoyette * or both.
1253 1.1.24.1 pgoyette */
1254 1.1.24.1 pgoyette if (work_claimed(&dw->work, wq)) {
1255 1.1.24.1 pgoyette /* On the queue. Remove and release. */
1256 1.1.24.1 pgoyette TAILQ_REMOVE(&wq->wq_dqueue, &dw->work,
1257 1.1.24.1 pgoyette work_entry);
1258 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel,
1259 1.1.24.1 pgoyette &dw->work, wq);
1260 1.1.24.1 pgoyette release_work(&dw->work, wq);
1261 1.1.24.1 pgoyette /* Can't dereference dw after this point. */
1262 1.1.24.1 pgoyette cancelled_p = true;
1263 1.1.24.1 pgoyette } else {
1264 1.1.24.1 pgoyette /* Not on the queue, so didn't cancel. */
1265 1.1.24.1 pgoyette cancelled_p = false;
1266 1.1.24.1 pgoyette }
1267 1.1.24.1 pgoyette /* If it's still running, wait for it to complete. */
1268 1.1.24.1 pgoyette if (wq->wq_current_work == &dw->work)
1269 1.1.24.1 pgoyette wait_for_current_work(&dw->work, wq);
1270 1.1.24.1 pgoyette break;
1271 1.1.24.1 pgoyette case DELAYED_WORK_SCHEDULED:
1272 1.1.24.1 pgoyette /*
1273 1.1.24.1 pgoyette * If it is scheduled, mark it cancelled and
1274 1.1.24.1 pgoyette * try to stop the callout before it starts.
1275 1.1.24.1 pgoyette *
1276 1.1.24.1 pgoyette * If it's too late and the callout has already
1277 1.1.24.1 pgoyette * begun to execute, we must wait for it to
1278 1.1.24.1 pgoyette * complete. But we got in soon enough to ask
1279 1.1.24.1 pgoyette * the callout not to run, so we successfully
1280 1.1.24.1 pgoyette * cancelled it in that case.
1281 1.1.24.1 pgoyette *
1282 1.1.24.1 pgoyette * If we stopped the callout before it started,
1283 1.1.24.1 pgoyette * then we must destroy the callout and
1284 1.1.24.1 pgoyette * dissociate it from the workqueue ourselves.
1285 1.1.24.1 pgoyette */
1286 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_CANCELLED;
1287 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq);
1288 1.1.24.1 pgoyette if (!callout_halt(&dw->dw_callout, &wq->wq_lock))
1289 1.1.24.1 pgoyette cancel_delayed_work_done(wq, dw);
1290 1.1.24.1 pgoyette cancelled_p = true;
1291 1.1.24.1 pgoyette break;
1292 1.1.24.1 pgoyette case DELAYED_WORK_RESCHEDULED:
1293 1.1.24.1 pgoyette /*
1294 1.1.24.1 pgoyette * If it is being rescheduled, the callout has
1295 1.1.24.1 pgoyette * already fired. We must ask it to cancel and
1296 1.1.24.1 pgoyette * wait for it to complete.
1297 1.1.24.1 pgoyette */
1298 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_CANCELLED;
1299 1.1.24.1 pgoyette dw->dw_resched = -1;
1300 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, cancel, &dw->work, wq);
1301 1.1.24.1 pgoyette (void)callout_halt(&dw->dw_callout, &wq->wq_lock);
1302 1.1.24.1 pgoyette cancelled_p = true;
1303 1.1.24.1 pgoyette break;
1304 1.1.24.1 pgoyette case DELAYED_WORK_CANCELLED:
1305 1.1.24.1 pgoyette /*
1306 1.1.24.1 pgoyette * If it is being cancelled, the callout has
1307 1.1.24.1 pgoyette * already fired. We need only wait for it to
1308 1.1.24.1 pgoyette * complete. Someone else, however, claims
1309 1.1.24.1 pgoyette * credit for cancelling it.
1310 1.1.24.1 pgoyette */
1311 1.1.24.1 pgoyette (void)callout_halt(&dw->dw_callout, &wq->wq_lock);
1312 1.1.24.1 pgoyette cancelled_p = false;
1313 1.1.24.1 pgoyette break;
1314 1.1 skrll default:
1315 1.1.24.1 pgoyette panic("invalid delayed work state: %d",
1316 1.1 skrll dw->dw_state);
1317 1.1 skrll }
1318 1.1 skrll }
1319 1.1.24.1 pgoyette mutex_exit(&wq->wq_lock);
1320 1.1.24.1 pgoyette
1321 1.1.24.1 pgoyette return cancelled_p;
1322 1.1.24.1 pgoyette }
1323 1.1.24.1 pgoyette
1324 1.1.24.1 pgoyette /*
1326 1.1.24.1 pgoyette * Flush
1327 1.1.24.1 pgoyette */
1328 1.1.24.1 pgoyette
1329 1.1.24.1 pgoyette /*
1330 1.1.24.1 pgoyette * flush_scheduled_work()
1331 1.1.24.1 pgoyette *
1332 1.1.24.1 pgoyette * Wait for all work queued on system_wq to complete. This does
1333 1.1 skrll * not include delayed work.
1334 1.1.24.1 pgoyette */
1335 1.1.24.1 pgoyette void
1336 1.1.24.1 pgoyette flush_scheduled_work(void)
1337 1.1.24.1 pgoyette {
1338 1.1.24.1 pgoyette
1339 1.1.24.1 pgoyette flush_workqueue(system_wq);
1340 1.1.24.1 pgoyette }
1341 1.1.24.1 pgoyette
1342 1.1.24.1 pgoyette /*
1343 1.1.24.1 pgoyette * flush_workqueue_locked(wq)
1344 1.1.24.1 pgoyette *
1345 1.1 skrll * Wait for all work queued on wq to complete. This does not
1346 1.1.24.1 pgoyette * include delayed work.
1347 1.1 skrll *
1348 1.1.24.1 pgoyette * Caller must hold wq's lock.
1349 1.1 skrll */
1350 1.1.24.1 pgoyette static void
1351 1.1 skrll flush_workqueue_locked(struct workqueue_struct *wq)
1352 1.1.24.1 pgoyette {
1353 1.1.24.1 pgoyette uint64_t gen;
1354 1.1 skrll
1355 1.1 skrll KASSERT(mutex_owned(&wq->wq_lock));
1356 1.1.24.1 pgoyette
1357 1.1.24.1 pgoyette /* Get the current generation number. */
1358 1.1 skrll gen = wq->wq_gen;
1359 1.1.24.1 pgoyette
1360 1.1.24.1 pgoyette /*
1361 1.1 skrll * If there's a batch of work in progress, we must wait for the
1362 1.1.24.1 pgoyette * worker thread to finish that batch.
1363 1.1.24.1 pgoyette */
1364 1.1.24.1 pgoyette if (wq->wq_current_work != NULL)
1365 1.1.24.1 pgoyette gen++;
1366 1.1.24.1 pgoyette
1367 1.1.24.1 pgoyette /*
1368 1.1.24.1 pgoyette * If there's any work yet to be claimed from the queue by the
1369 1.1 skrll * worker thread, we must wait for it to finish one more batch
1370 1.1.24.1 pgoyette * too.
1371 1.1.24.1 pgoyette */
1372 1.1.24.1 pgoyette if (!TAILQ_EMPTY(&wq->wq_queue) || !TAILQ_EMPTY(&wq->wq_dqueue))
1373 1.1.24.1 pgoyette gen++;
1374 1.1.24.1 pgoyette
1375 1.1 skrll /* Wait until the generation number has caught up. */
1376 1.1 skrll SDT_PROBE1(sdt, linux, work, flush__start, wq);
1377 1.1.24.1 pgoyette while (wq->wq_gen < gen)
1378 1.1.24.1 pgoyette cv_wait(&wq->wq_cv, &wq->wq_lock);
1379 1.1.24.1 pgoyette SDT_PROBE1(sdt, linux, work, flush__done, wq);
1380 1.1.24.1 pgoyette }
1381 1.1.24.1 pgoyette
1382 1.1.24.1 pgoyette /*
1383 1.1.24.1 pgoyette * flush_workqueue(wq)
1384 1.1.24.1 pgoyette *
1385 1.1.24.1 pgoyette * Wait for all work queued on wq to complete. This does not
1386 1.1.24.1 pgoyette * include delayed work.
1387 1.1.24.1 pgoyette */
1388 1.1.24.1 pgoyette void
1389 1.1.24.1 pgoyette flush_workqueue(struct workqueue_struct *wq)
1390 1.1.24.1 pgoyette {
1391 1.1.24.1 pgoyette
1392 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
1393 1.1.24.1 pgoyette flush_workqueue_locked(wq);
1394 1.1.24.1 pgoyette mutex_exit(&wq->wq_lock);
1395 1.1.24.1 pgoyette }
1396 1.1.24.1 pgoyette
1397 1.1.24.1 pgoyette /*
1398 1.1.24.1 pgoyette * flush_work(work)
1399 1.1.24.1 pgoyette *
1400 1.1 skrll * If work is queued or currently executing, wait for it to
1401 1.1 skrll * complete.
1402 1.1 skrll */
1403 1.1.24.1 pgoyette void
1404 1.1.24.1 pgoyette flush_work(struct work_struct *work)
1405 1.1.24.1 pgoyette {
1406 1.1 skrll struct workqueue_struct *wq;
1407 1.1.24.1 pgoyette
1408 1.1 skrll /* If there's no workqueue, nothing to flush. */
1409 1.1 skrll if ((wq = work_queue(work)) == NULL)
1410 1.1.24.1 pgoyette return;
1411 1.1.24.1 pgoyette
1412 1.1.24.1 pgoyette flush_workqueue(wq);
1413 1.1.24.1 pgoyette }
1414 1.1.24.1 pgoyette
1415 1.1.24.1 pgoyette /*
1416 1.1.24.1 pgoyette * flush_delayed_work(dw)
1417 1.1.24.1 pgoyette *
1418 1.1.24.1 pgoyette * If dw is scheduled to run after a delay, queue it immediately
1419 1.1 skrll * instead. Then, if dw is queued or currently executing, wait
1420 1.1 skrll * for it to complete.
1421 1.1 skrll */
1422 1.1.24.1 pgoyette void
1423 1.1.24.1 pgoyette flush_delayed_work(struct delayed_work *dw)
1424 1.1.24.1 pgoyette {
1425 1.1 skrll struct workqueue_struct *wq;
1426 1.1 skrll
1427 1.1.24.1 pgoyette /* If there's no workqueue, nothing to flush. */
1428 1.1.24.1 pgoyette if ((wq = work_queue(&dw->work)) == NULL)
1429 1.1.24.1 pgoyette return;
1430 1.1.24.1 pgoyette
1431 1.1.24.1 pgoyette mutex_enter(&wq->wq_lock);
1432 1.1.24.1 pgoyette if (__predict_false(work_queue(&dw->work) != wq)) {
1433 1.1 skrll /*
1434 1.1.24.1 pgoyette * Moved off the queue already (and possibly to another
1435 1.1.24.1 pgoyette * queue, though that would be ill-advised), so it must
1436 1.1.24.1 pgoyette * have completed, and we have nothing more to do.
1437 1.1.24.1 pgoyette */
1438 1.1.24.1 pgoyette } else {
1439 1.1.24.1 pgoyette switch (dw->dw_state) {
1440 1.1.24.1 pgoyette case DELAYED_WORK_IDLE:
1441 1.1.24.1 pgoyette /*
1442 1.1.24.1 pgoyette * It has a workqueue assigned and the callout
1443 1.1.24.1 pgoyette * is idle, so it must be in progress or on the
1444 1.1.24.1 pgoyette * queue. In that case, we'll wait for it to
1445 1.1.24.1 pgoyette * complete.
1446 1.1.24.1 pgoyette */
1447 1.1.24.1 pgoyette break;
1448 1.1.24.1 pgoyette case DELAYED_WORK_SCHEDULED:
1449 1.1.24.1 pgoyette case DELAYED_WORK_RESCHEDULED:
1450 1.1.24.1 pgoyette case DELAYED_WORK_CANCELLED:
1451 1.1.24.1 pgoyette /*
1452 1.1.24.1 pgoyette * The callout is scheduled, and may have even
1453 1.1.24.1 pgoyette * started. Mark it as scheduled so that if
1454 1.1.24.1 pgoyette * the callout has fired it will queue the work
1455 1.1.24.1 pgoyette * itself. Try to stop the callout -- if we
1456 1.1.24.1 pgoyette * can, queue the work now; if we can't, wait
1457 1.1.24.1 pgoyette * for the callout to complete, which entails
1458 1.1.24.1 pgoyette * queueing it.
1459 1.1.24.1 pgoyette */
1460 1.1.24.1 pgoyette dw->dw_state = DELAYED_WORK_SCHEDULED;
1461 1.1.24.1 pgoyette if (!callout_halt(&dw->dw_callout, &wq->wq_lock)) {
1462 1.1.24.1 pgoyette /*
1463 1.1.24.1 pgoyette * We stopped it before it ran. No
1464 1.1.24.1 pgoyette * state change in the interim is
1465 1.1.24.1 pgoyette * possible. Destroy the callout and
1466 1.1.24.1 pgoyette * queue it ourselves.
1467 1.1.24.1 pgoyette */
1468 1.1.24.1 pgoyette KASSERT(dw->dw_state ==
1469 1.1.24.1 pgoyette DELAYED_WORK_SCHEDULED);
1470 1.1.24.1 pgoyette dw_callout_destroy(wq, dw);
1471 1.1.24.1 pgoyette TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work,
1472 1.1.24.1 pgoyette work_entry);
1473 1.1.24.1 pgoyette cv_broadcast(&wq->wq_cv);
1474 1.1.24.1 pgoyette SDT_PROBE2(sdt, linux, work, queue,
1475 1.1.24.1 pgoyette &dw->work, wq);
1476 1.1.24.1 pgoyette }
1477 1.1.24.1 pgoyette break;
1478 1.1.24.1 pgoyette default:
1479 1.1.24.1 pgoyette panic("invalid delayed work state: %d", dw->dw_state);
1480 1.1.24.1 pgoyette }
1481 1.1 skrll /*
1482 1.1 skrll * Waiting for the whole queue to flush is overkill,
1483 1.1 skrll * but doesn't hurt.
1484 */
1485 flush_workqueue_locked(wq);
1486 }
1487 mutex_exit(&wq->wq_lock);
1488 }
1489