Home | History | Annotate | Line # | Download | only in linux
linux_work.c revision 1.1.24.1
      1  1.1.24.1  pgoyette /*	$NetBSD: linux_work.c,v 1.1.24.1 2018/09/06 06:56:08 pgoyette Exp $	*/
      2       1.1     skrll 
      3       1.1     skrll /*-
      4  1.1.24.1  pgoyette  * Copyright (c) 2018 The NetBSD Foundation, Inc.
      5       1.1     skrll  * All rights reserved.
      6       1.1     skrll  *
      7       1.1     skrll  * This code is derived from software contributed to The NetBSD Foundation
      8       1.1     skrll  * by Taylor R. Campbell.
      9       1.1     skrll  *
     10       1.1     skrll  * Redistribution and use in source and binary forms, with or without
     11       1.1     skrll  * modification, are permitted provided that the following conditions
     12       1.1     skrll  * are met:
     13       1.1     skrll  * 1. Redistributions of source code must retain the above copyright
     14       1.1     skrll  *    notice, this list of conditions and the following disclaimer.
     15       1.1     skrll  * 2. Redistributions in binary form must reproduce the above copyright
     16       1.1     skrll  *    notice, this list of conditions and the following disclaimer in the
     17       1.1     skrll  *    documentation and/or other materials provided with the distribution.
     18       1.1     skrll  *
     19       1.1     skrll  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20       1.1     skrll  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21       1.1     skrll  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22       1.1     skrll  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23       1.1     skrll  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24       1.1     skrll  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25       1.1     skrll  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26       1.1     skrll  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27       1.1     skrll  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28       1.1     skrll  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29       1.1     skrll  * POSSIBILITY OF SUCH DAMAGE.
     30       1.1     skrll  */
     31       1.1     skrll 
     32       1.1     skrll #include <sys/cdefs.h>
     33  1.1.24.1  pgoyette __KERNEL_RCSID(0, "$NetBSD: linux_work.c,v 1.1.24.1 2018/09/06 06:56:08 pgoyette Exp $");
     34       1.1     skrll 
     35       1.1     skrll #include <sys/types.h>
     36       1.1     skrll #include <sys/atomic.h>
     37       1.1     skrll #include <sys/callout.h>
     38       1.1     skrll #include <sys/condvar.h>
     39       1.1     skrll #include <sys/errno.h>
     40       1.1     skrll #include <sys/kmem.h>
     41  1.1.24.1  pgoyette #include <sys/kthread.h>
     42  1.1.24.1  pgoyette #include <sys/lwp.h>
     43       1.1     skrll #include <sys/mutex.h>
     44       1.1     skrll #include <sys/queue.h>
     45  1.1.24.1  pgoyette #include <sys/sdt.h>
     46       1.1     skrll 
     47       1.1     skrll #include <linux/workqueue.h>
     48       1.1     skrll 
     49  1.1.24.1  pgoyette TAILQ_HEAD(work_head, work_struct);
     50  1.1.24.1  pgoyette TAILQ_HEAD(dwork_head, delayed_work);
     51       1.1     skrll 
     52       1.1     skrll struct workqueue_struct {
     53  1.1.24.1  pgoyette 	kmutex_t		wq_lock;
     54  1.1.24.1  pgoyette 	kcondvar_t		wq_cv;
     55  1.1.24.1  pgoyette 	struct dwork_head	wq_delayed; /* delayed work scheduled */
     56  1.1.24.1  pgoyette 	struct work_head	wq_queue;   /* work to run */
     57  1.1.24.1  pgoyette 	struct work_head	wq_dqueue;  /* delayed work to run now */
     58  1.1.24.1  pgoyette 	struct work_struct	*wq_current_work;
     59  1.1.24.1  pgoyette 	int			wq_flags;
     60  1.1.24.1  pgoyette 	bool			wq_dying;
     61  1.1.24.1  pgoyette 	uint64_t		wq_gen;
     62  1.1.24.1  pgoyette 	struct lwp		*wq_lwp;
     63       1.1     skrll };
     64       1.1     skrll 
     65  1.1.24.1  pgoyette static void __dead	linux_workqueue_thread(void *);
     66  1.1.24.1  pgoyette static void		linux_workqueue_timeout(void *);
     67  1.1.24.1  pgoyette static bool		work_claimed(struct work_struct *,
     68  1.1.24.1  pgoyette 			    struct workqueue_struct *);
     69  1.1.24.1  pgoyette static struct workqueue_struct *
     70  1.1.24.1  pgoyette 			work_queue(struct work_struct *);
     71  1.1.24.1  pgoyette static bool		acquire_work(struct work_struct *,
     72  1.1.24.1  pgoyette 			    struct workqueue_struct *);
     73  1.1.24.1  pgoyette static void		release_work(struct work_struct *,
     74  1.1.24.1  pgoyette 			    struct workqueue_struct *);
     75  1.1.24.1  pgoyette static void		wait_for_current_work(struct work_struct *,
     76  1.1.24.1  pgoyette 			    struct workqueue_struct *);
     77  1.1.24.1  pgoyette static void		dw_callout_init(struct workqueue_struct *,
     78  1.1.24.1  pgoyette 			    struct delayed_work *);
     79  1.1.24.1  pgoyette static void		dw_callout_destroy(struct workqueue_struct *,
     80  1.1.24.1  pgoyette 			    struct delayed_work *);
     81  1.1.24.1  pgoyette static void		cancel_delayed_work_done(struct workqueue_struct *,
     82  1.1.24.1  pgoyette 			    struct delayed_work *);
     83  1.1.24.1  pgoyette 
     84  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, acquire,
     85  1.1.24.1  pgoyette     "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
     86  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, release,
     87  1.1.24.1  pgoyette     "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
     88  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, queue,
     89  1.1.24.1  pgoyette     "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
     90  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, cancel,
     91  1.1.24.1  pgoyette     "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
     92  1.1.24.1  pgoyette SDT_PROBE_DEFINE3(sdt, linux, work, schedule,
     93  1.1.24.1  pgoyette     "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/,
     94  1.1.24.1  pgoyette     "unsigned long"/*ticks*/);
     95  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, timer,
     96  1.1.24.1  pgoyette     "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/);
     97  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, wait__start,
     98  1.1.24.1  pgoyette     "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/);
     99  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, wait__done,
    100  1.1.24.1  pgoyette     "struct delayed_work *"/*dw*/, "struct workqueue_struct *"/*wq*/);
    101  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, run,
    102  1.1.24.1  pgoyette     "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
    103  1.1.24.1  pgoyette SDT_PROBE_DEFINE2(sdt, linux, work, done,
    104  1.1.24.1  pgoyette     "struct work_struct *"/*work*/, "struct workqueue_struct *"/*wq*/);
    105  1.1.24.1  pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, batch__start,
    106  1.1.24.1  pgoyette     "struct workqueue_struct *"/*wq*/);
    107  1.1.24.1  pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, batch__done,
    108  1.1.24.1  pgoyette     "struct workqueue_struct *"/*wq*/);
    109  1.1.24.1  pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, flush__start,
    110  1.1.24.1  pgoyette     "struct workqueue_struct *"/*wq*/);
    111  1.1.24.1  pgoyette SDT_PROBE_DEFINE1(sdt, linux, work, flush__done,
    112  1.1.24.1  pgoyette     "struct workqueue_struct *"/*wq*/);
    113  1.1.24.1  pgoyette 
    114  1.1.24.1  pgoyette static specificdata_key_t workqueue_key __read_mostly;
    115  1.1.24.1  pgoyette 
    116  1.1.24.1  pgoyette struct workqueue_struct	*system_wq __read_mostly;
    117  1.1.24.1  pgoyette struct workqueue_struct	*system_long_wq __read_mostly;
    118  1.1.24.1  pgoyette struct workqueue_struct	*system_power_efficient_wq __read_mostly;
    119       1.1     skrll 
    120  1.1.24.1  pgoyette static inline uintptr_t
    121  1.1.24.1  pgoyette atomic_cas_uintptr(volatile uintptr_t *p, uintptr_t old, uintptr_t new)
    122  1.1.24.1  pgoyette {
    123       1.1     skrll 
    124  1.1.24.1  pgoyette 	return (uintptr_t)atomic_cas_ptr(p, (void *)old, (void *)new);
    125  1.1.24.1  pgoyette }
    126  1.1.24.1  pgoyette 
    127  1.1.24.1  pgoyette /*
    128  1.1.24.1  pgoyette  * linux_workqueue_init()
    129  1.1.24.1  pgoyette  *
    130  1.1.24.1  pgoyette  *	Initialize the Linux workqueue subsystem.  Return 0 on success,
    131  1.1.24.1  pgoyette  *	NetBSD error on failure.
    132  1.1.24.1  pgoyette  */
    133       1.1     skrll int
    134       1.1     skrll linux_workqueue_init(void)
    135       1.1     skrll {
    136  1.1.24.1  pgoyette 	int error;
    137  1.1.24.1  pgoyette 
    138  1.1.24.1  pgoyette 	error = lwp_specific_key_create(&workqueue_key, NULL);
    139  1.1.24.1  pgoyette 	if (error)
    140  1.1.24.1  pgoyette 		goto fail0;
    141       1.1     skrll 
    142       1.1     skrll 	system_wq = alloc_ordered_workqueue("lnxsyswq", 0);
    143  1.1.24.1  pgoyette 	if (system_wq == NULL) {
    144  1.1.24.1  pgoyette 		error = ENOMEM;
    145  1.1.24.1  pgoyette 		goto fail1;
    146  1.1.24.1  pgoyette 	}
    147  1.1.24.1  pgoyette 
    148  1.1.24.1  pgoyette 	system_long_wq = alloc_ordered_workqueue("lnxlngwq", 0);
    149  1.1.24.1  pgoyette 	if (system_long_wq == NULL) {
    150  1.1.24.1  pgoyette 		error = ENOMEM;
    151  1.1.24.1  pgoyette 		goto fail2;
    152  1.1.24.1  pgoyette 	}
    153  1.1.24.1  pgoyette 
    154  1.1.24.1  pgoyette 	system_power_efficient_wq = alloc_ordered_workqueue("lnxpwrwq", 0);
    155  1.1.24.1  pgoyette 	if (system_long_wq == NULL) {
    156  1.1.24.1  pgoyette 		error = ENOMEM;
    157  1.1.24.1  pgoyette 		goto fail3;
    158  1.1.24.1  pgoyette 	}
    159       1.1     skrll 
    160       1.1     skrll 	return 0;
    161  1.1.24.1  pgoyette 
    162  1.1.24.1  pgoyette fail4: __unused
    163  1.1.24.1  pgoyette 	destroy_workqueue(system_power_efficient_wq);
    164  1.1.24.1  pgoyette fail3:	destroy_workqueue(system_long_wq);
    165  1.1.24.1  pgoyette fail2:	destroy_workqueue(system_wq);
    166  1.1.24.1  pgoyette fail1:	lwp_specific_key_delete(workqueue_key);
    167  1.1.24.1  pgoyette fail0:	KASSERT(error);
    168  1.1.24.1  pgoyette 	return error;
    169       1.1     skrll }
    170       1.1     skrll 
    171  1.1.24.1  pgoyette /*
    172  1.1.24.1  pgoyette  * linux_workqueue_fini()
    173  1.1.24.1  pgoyette  *
    174  1.1.24.1  pgoyette  *	Destroy the Linux workqueue subsystem.  Never fails.
    175  1.1.24.1  pgoyette  */
    176       1.1     skrll void
    177       1.1     skrll linux_workqueue_fini(void)
    178       1.1     skrll {
    179  1.1.24.1  pgoyette 
    180  1.1.24.1  pgoyette 	destroy_workqueue(system_power_efficient_wq);
    181  1.1.24.1  pgoyette 	destroy_workqueue(system_long_wq);
    182       1.1     skrll 	destroy_workqueue(system_wq);
    183  1.1.24.1  pgoyette 	lwp_specific_key_delete(workqueue_key);
    184       1.1     skrll }
    185       1.1     skrll 
    186       1.1     skrll /*
    188       1.1     skrll  * Workqueues
    189       1.1     skrll  */
    190  1.1.24.1  pgoyette 
    191  1.1.24.1  pgoyette /*
    192  1.1.24.1  pgoyette  * alloc_ordered_workqueue(name, flags)
    193  1.1.24.1  pgoyette  *
    194  1.1.24.1  pgoyette  *	Create a workqueue of the given name.  No flags are currently
    195  1.1.24.1  pgoyette  *	defined.  Return NULL on failure, pointer to struct
    196  1.1.24.1  pgoyette  *	workqueue_struct object on success.
    197       1.1     skrll  */
    198  1.1.24.1  pgoyette struct workqueue_struct *
    199       1.1     skrll alloc_ordered_workqueue(const char *name, int flags)
    200       1.1     skrll {
    201       1.1     skrll 	struct workqueue_struct *wq;
    202       1.1     skrll 	int error;
    203  1.1.24.1  pgoyette 
    204       1.1     skrll 	KASSERT(flags == 0);
    205  1.1.24.1  pgoyette 
    206       1.1     skrll 	wq = kmem_zalloc(sizeof(*wq), KM_SLEEP);
    207       1.1     skrll 
    208       1.1     skrll 	mutex_init(&wq->wq_lock, MUTEX_DEFAULT, IPL_VM);
    209       1.1     skrll 	cv_init(&wq->wq_cv, name);
    210  1.1.24.1  pgoyette 	TAILQ_INIT(&wq->wq_delayed);
    211  1.1.24.1  pgoyette 	TAILQ_INIT(&wq->wq_queue);
    212       1.1     skrll 	TAILQ_INIT(&wq->wq_dqueue);
    213  1.1.24.1  pgoyette 	wq->wq_current_work = NULL;
    214  1.1.24.1  pgoyette 	wq->wq_flags = 0;
    215  1.1.24.1  pgoyette 	wq->wq_dying = false;
    216  1.1.24.1  pgoyette 	wq->wq_gen = 0;
    217  1.1.24.1  pgoyette 	wq->wq_lwp = NULL;
    218  1.1.24.1  pgoyette 
    219  1.1.24.1  pgoyette 	error = kthread_create(PRI_NONE,
    220  1.1.24.1  pgoyette 	    KTHREAD_MPSAFE|KTHREAD_TS|KTHREAD_MUSTJOIN, NULL,
    221  1.1.24.1  pgoyette 	    &linux_workqueue_thread, wq, &wq->wq_lwp, "%s", name);
    222  1.1.24.1  pgoyette 	if (error)
    223       1.1     skrll 		goto fail0;
    224       1.1     skrll 
    225  1.1.24.1  pgoyette 	return wq;
    226  1.1.24.1  pgoyette 
    227  1.1.24.1  pgoyette fail0:	KASSERT(TAILQ_EMPTY(&wq->wq_dqueue));
    228  1.1.24.1  pgoyette 	KASSERT(TAILQ_EMPTY(&wq->wq_queue));
    229  1.1.24.1  pgoyette 	KASSERT(TAILQ_EMPTY(&wq->wq_delayed));
    230  1.1.24.1  pgoyette 	cv_destroy(&wq->wq_cv);
    231  1.1.24.1  pgoyette 	mutex_destroy(&wq->wq_lock);
    232  1.1.24.1  pgoyette 	kmem_free(wq, sizeof(*wq));
    233       1.1     skrll 	return NULL;
    234       1.1     skrll }
    235  1.1.24.1  pgoyette 
    236  1.1.24.1  pgoyette /*
    237  1.1.24.1  pgoyette  * destroy_workqueue(wq)
    238  1.1.24.1  pgoyette  *
    239  1.1.24.1  pgoyette  *	Destroy a workqueue created with wq.  Cancel any pending
    240  1.1.24.1  pgoyette  *	delayed work.  Wait for all queued work to complete.
    241  1.1.24.1  pgoyette  *
    242  1.1.24.1  pgoyette  *	May sleep.
    243       1.1     skrll  */
    244       1.1     skrll void
    245       1.1     skrll destroy_workqueue(struct workqueue_struct *wq)
    246       1.1     skrll {
    247       1.1     skrll 
    248  1.1.24.1  pgoyette 	/*
    249  1.1.24.1  pgoyette 	 * Cancel all delayed work.  We do this first because any
    250  1.1.24.1  pgoyette 	 * delayed work that that has already timed out, which we can't
    251       1.1     skrll 	 * cancel, may have queued new work.
    252  1.1.24.1  pgoyette 	 */
    253  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    254  1.1.24.1  pgoyette 	while (!TAILQ_EMPTY(&wq->wq_delayed)) {
    255       1.1     skrll 		struct delayed_work *const dw = TAILQ_FIRST(&wq->wq_delayed);
    256  1.1.24.1  pgoyette 
    257  1.1.24.1  pgoyette 		KASSERT(work_queue(&dw->work) == wq);
    258  1.1.24.1  pgoyette 		KASSERTMSG((dw->dw_state == DELAYED_WORK_SCHEDULED ||
    259  1.1.24.1  pgoyette 			dw->dw_state == DELAYED_WORK_RESCHEDULED ||
    260  1.1.24.1  pgoyette 			dw->dw_state == DELAYED_WORK_CANCELLED),
    261  1.1.24.1  pgoyette 		    "delayed work %p in bad state: %d",
    262       1.1     skrll 		    dw, dw->dw_state);
    263  1.1.24.1  pgoyette 
    264  1.1.24.1  pgoyette 		/*
    265  1.1.24.1  pgoyette 		 * Mark it cancelled and try to stop the callout before
    266  1.1.24.1  pgoyette 		 * it starts.
    267  1.1.24.1  pgoyette 		 *
    268  1.1.24.1  pgoyette 		 * If it's too late and the callout has already begun
    269  1.1.24.1  pgoyette 		 * to execute, then it will notice that we asked to
    270  1.1.24.1  pgoyette 		 * cancel it and remove itself from the queue before
    271  1.1.24.1  pgoyette 		 * returning.
    272  1.1.24.1  pgoyette 		 *
    273  1.1.24.1  pgoyette 		 * If we stopped the callout before it started,
    274  1.1.24.1  pgoyette 		 * however, then we can safely destroy the callout and
    275  1.1.24.1  pgoyette 		 * dissociate it from the workqueue ourselves.
    276  1.1.24.1  pgoyette 		 */
    277  1.1.24.1  pgoyette 		SDT_PROBE2(sdt, linux, work, cancel,  &dw->work, wq);
    278  1.1.24.1  pgoyette 		dw->dw_state = DELAYED_WORK_CANCELLED;
    279  1.1.24.1  pgoyette 		if (!callout_halt(&dw->dw_callout, &wq->wq_lock))
    280       1.1     skrll 			cancel_delayed_work_done(wq, dw);
    281  1.1.24.1  pgoyette 	}
    282       1.1     skrll 	mutex_exit(&wq->wq_lock);
    283       1.1     skrll 
    284  1.1.24.1  pgoyette 	/*
    285       1.1     skrll 	 * At this point, no new work can be put on the queue.
    286       1.1     skrll 	 */
    287  1.1.24.1  pgoyette 
    288  1.1.24.1  pgoyette 	/* Tell the thread to exit.  */
    289  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    290  1.1.24.1  pgoyette 	wq->wq_dying = true;
    291  1.1.24.1  pgoyette 	cv_broadcast(&wq->wq_cv);
    292  1.1.24.1  pgoyette 	mutex_exit(&wq->wq_lock);
    293  1.1.24.1  pgoyette 
    294  1.1.24.1  pgoyette 	/* Wait for it to exit.  */
    295  1.1.24.1  pgoyette 	(void)kthread_join(wq->wq_lwp);
    296  1.1.24.1  pgoyette 
    297  1.1.24.1  pgoyette 	KASSERT(wq->wq_dying);
    298  1.1.24.1  pgoyette 	KASSERT(wq->wq_flags == 0);
    299  1.1.24.1  pgoyette 	KASSERT(wq->wq_current_work == NULL);
    300  1.1.24.1  pgoyette 	KASSERT(TAILQ_EMPTY(&wq->wq_dqueue));
    301  1.1.24.1  pgoyette 	KASSERT(TAILQ_EMPTY(&wq->wq_queue));
    302       1.1     skrll 	KASSERT(TAILQ_EMPTY(&wq->wq_delayed));
    303       1.1     skrll 	cv_destroy(&wq->wq_cv);
    304       1.1     skrll 	mutex_destroy(&wq->wq_lock);
    305       1.1     skrll 
    306       1.1     skrll 	kmem_free(wq, sizeof(*wq));
    307       1.1     skrll }
    308       1.1     skrll 
    309  1.1.24.1  pgoyette /*
    311       1.1     skrll  * Work thread and callout
    312  1.1.24.1  pgoyette  */
    313  1.1.24.1  pgoyette 
    314  1.1.24.1  pgoyette /*
    315  1.1.24.1  pgoyette  * linux_workqueue_thread(cookie)
    316  1.1.24.1  pgoyette  *
    317  1.1.24.1  pgoyette  *	Main function for a workqueue's worker thread.  Waits until
    318  1.1.24.1  pgoyette  *	there is work queued, grabs a batch of work off the queue,
    319  1.1.24.1  pgoyette  *	executes it all, bumps the generation number, and repeats,
    320  1.1.24.1  pgoyette  *	until dying.
    321  1.1.24.1  pgoyette  */
    322       1.1     skrll static void __dead
    323  1.1.24.1  pgoyette linux_workqueue_thread(void *cookie)
    324  1.1.24.1  pgoyette {
    325  1.1.24.1  pgoyette 	struct workqueue_struct *const wq = cookie;
    326  1.1.24.1  pgoyette 	struct work_head queue, dqueue;
    327       1.1     skrll 	struct work_head *const q[2] = { &queue, &dqueue };
    328  1.1.24.1  pgoyette 	unsigned i;
    329       1.1     skrll 
    330  1.1.24.1  pgoyette 	lwp_setspecific(workqueue_key, wq);
    331  1.1.24.1  pgoyette 
    332  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    333  1.1.24.1  pgoyette 	for (;;) {
    334  1.1.24.1  pgoyette 		/*
    335  1.1.24.1  pgoyette 		 * Wait until there's activity.  If there's no work and
    336  1.1.24.1  pgoyette 		 * we're dying, stop here.
    337  1.1.24.1  pgoyette 		 */
    338  1.1.24.1  pgoyette 		if (TAILQ_EMPTY(&wq->wq_queue) &&
    339  1.1.24.1  pgoyette 		    TAILQ_EMPTY(&wq->wq_dqueue)) {
    340  1.1.24.1  pgoyette 			if (wq->wq_dying)
    341  1.1.24.1  pgoyette 				break;
    342  1.1.24.1  pgoyette 			cv_wait(&wq->wq_cv, &wq->wq_lock);
    343       1.1     skrll 			continue;
    344  1.1.24.1  pgoyette 		}
    345  1.1.24.1  pgoyette 
    346  1.1.24.1  pgoyette 		/* Grab a batch of work off the queue.  */
    347  1.1.24.1  pgoyette 		SDT_PROBE1(sdt, linux, work, batch__start,  wq);
    348  1.1.24.1  pgoyette 		TAILQ_INIT(&queue);
    349  1.1.24.1  pgoyette 		TAILQ_INIT(&dqueue);
    350  1.1.24.1  pgoyette 		TAILQ_CONCAT(&queue, &wq->wq_queue, work_entry);
    351  1.1.24.1  pgoyette 		TAILQ_CONCAT(&dqueue, &wq->wq_dqueue, work_entry);
    352  1.1.24.1  pgoyette 
    353  1.1.24.1  pgoyette 		/* Process each work item in the batch.  */
    354  1.1.24.1  pgoyette 		for (i = 0; i < 2; i++) {
    355  1.1.24.1  pgoyette 			while (!TAILQ_EMPTY(q[i])) {
    356  1.1.24.1  pgoyette 				struct work_struct *work = TAILQ_FIRST(q[i]);
    357  1.1.24.1  pgoyette 				void (*func)(struct work_struct *);
    358  1.1.24.1  pgoyette 
    359  1.1.24.1  pgoyette 				KASSERT(work_queue(work) == wq);
    360  1.1.24.1  pgoyette 				KASSERT(work_claimed(work, wq));
    361  1.1.24.1  pgoyette 				KASSERTMSG((q[i] != &dqueue ||
    362  1.1.24.1  pgoyette 					container_of(work, struct delayed_work,
    363  1.1.24.1  pgoyette 					    work)->dw_state ==
    364  1.1.24.1  pgoyette 					DELAYED_WORK_IDLE),
    365  1.1.24.1  pgoyette 				    "delayed work %p queued and scheduled",
    366  1.1.24.1  pgoyette 				    work);
    367  1.1.24.1  pgoyette 
    368  1.1.24.1  pgoyette 				TAILQ_REMOVE(q[i], work, work_entry);
    369  1.1.24.1  pgoyette 				KASSERT(wq->wq_current_work == NULL);
    370  1.1.24.1  pgoyette 				wq->wq_current_work = work;
    371  1.1.24.1  pgoyette 				func = work->func;
    372  1.1.24.1  pgoyette 				release_work(work, wq);
    373  1.1.24.1  pgoyette 				/* Can't dereference work after this point.  */
    374  1.1.24.1  pgoyette 
    375  1.1.24.1  pgoyette 				mutex_exit(&wq->wq_lock);
    376  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, run,  work, wq);
    377  1.1.24.1  pgoyette 				(*func)(work);
    378  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, done,  work, wq);
    379  1.1.24.1  pgoyette 				mutex_enter(&wq->wq_lock);
    380  1.1.24.1  pgoyette 
    381  1.1.24.1  pgoyette 				KASSERT(wq->wq_current_work == work);
    382  1.1.24.1  pgoyette 				wq->wq_current_work = NULL;
    383  1.1.24.1  pgoyette 				cv_broadcast(&wq->wq_cv);
    384       1.1     skrll 			}
    385  1.1.24.1  pgoyette 		}
    386  1.1.24.1  pgoyette 
    387  1.1.24.1  pgoyette 		/* Notify flush that we've completed a batch of work.  */
    388  1.1.24.1  pgoyette 		wq->wq_gen++;
    389  1.1.24.1  pgoyette 		cv_broadcast(&wq->wq_cv);
    390  1.1.24.1  pgoyette 		SDT_PROBE1(sdt, linux, work, batch__done,  wq);
    391  1.1.24.1  pgoyette 	}
    392  1.1.24.1  pgoyette 	mutex_exit(&wq->wq_lock);
    393       1.1     skrll 
    394       1.1     skrll 	kthread_exit(0);
    395  1.1.24.1  pgoyette }
    396  1.1.24.1  pgoyette 
    397  1.1.24.1  pgoyette /*
    398  1.1.24.1  pgoyette  * linux_workqueue_timeout(cookie)
    399  1.1.24.1  pgoyette  *
    400  1.1.24.1  pgoyette  *	Delayed work timeout callback.
    401  1.1.24.1  pgoyette  *
    402  1.1.24.1  pgoyette  *	- If scheduled, queue it.
    403  1.1.24.1  pgoyette  *	- If rescheduled, callout_schedule ourselves again.
    404  1.1.24.1  pgoyette  *	- If cancelled, destroy the callout and release the work from
    405  1.1.24.1  pgoyette  *        the workqueue.
    406  1.1.24.1  pgoyette  */
    407       1.1     skrll static void
    408  1.1.24.1  pgoyette linux_workqueue_timeout(void *cookie)
    409  1.1.24.1  pgoyette {
    410       1.1     skrll 	struct delayed_work *const dw = cookie;
    411  1.1.24.1  pgoyette 	struct workqueue_struct *const wq = work_queue(&dw->work);
    412  1.1.24.1  pgoyette 
    413  1.1.24.1  pgoyette 	KASSERTMSG(wq != NULL,
    414       1.1     skrll 	    "delayed work %p state %d resched %d",
    415  1.1.24.1  pgoyette 	    dw, dw->dw_state, dw->dw_resched);
    416       1.1     skrll 
    417  1.1.24.1  pgoyette 	SDT_PROBE2(sdt, linux, work, timer,  dw, wq);
    418  1.1.24.1  pgoyette 
    419  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    420  1.1.24.1  pgoyette 	KASSERT(work_queue(&dw->work) == wq);
    421  1.1.24.1  pgoyette 	switch (dw->dw_state) {
    422  1.1.24.1  pgoyette 	case DELAYED_WORK_IDLE:
    423  1.1.24.1  pgoyette 		panic("delayed work callout uninitialized: %p", dw);
    424  1.1.24.1  pgoyette 	case DELAYED_WORK_SCHEDULED:
    425  1.1.24.1  pgoyette 		dw_callout_destroy(wq, dw);
    426  1.1.24.1  pgoyette 		TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work, work_entry);
    427  1.1.24.1  pgoyette 		cv_broadcast(&wq->wq_cv);
    428  1.1.24.1  pgoyette 		SDT_PROBE2(sdt, linux, work, queue,  &dw->work, wq);
    429  1.1.24.1  pgoyette 		break;
    430  1.1.24.1  pgoyette 	case DELAYED_WORK_RESCHEDULED:
    431  1.1.24.1  pgoyette 		KASSERT(dw->dw_resched >= 0);
    432  1.1.24.1  pgoyette 		callout_schedule(&dw->dw_callout, dw->dw_resched);
    433  1.1.24.1  pgoyette 		dw->dw_state = DELAYED_WORK_SCHEDULED;
    434  1.1.24.1  pgoyette 		dw->dw_resched = -1;
    435  1.1.24.1  pgoyette 		break;
    436  1.1.24.1  pgoyette 	case DELAYED_WORK_CANCELLED:
    437  1.1.24.1  pgoyette 		cancel_delayed_work_done(wq, dw);
    438  1.1.24.1  pgoyette 		/* Can't dereference dw after this point.  */
    439  1.1.24.1  pgoyette 		goto out;
    440  1.1.24.1  pgoyette 	default:
    441  1.1.24.1  pgoyette 		panic("delayed work callout in bad state: %p", dw);
    442  1.1.24.1  pgoyette 	}
    443  1.1.24.1  pgoyette 	KASSERT(dw->dw_state == DELAYED_WORK_IDLE ||
    444       1.1     skrll 	    dw->dw_state == DELAYED_WORK_SCHEDULED);
    445       1.1     skrll out:	mutex_exit(&wq->wq_lock);
    446  1.1.24.1  pgoyette }
    447  1.1.24.1  pgoyette 
    448  1.1.24.1  pgoyette /*
    449  1.1.24.1  pgoyette  * current_work()
    450  1.1.24.1  pgoyette  *
    451  1.1.24.1  pgoyette  *	If in a workqueue worker thread, return the work it is
    452  1.1.24.1  pgoyette  *	currently executing.  Otherwise return NULL.
    453  1.1.24.1  pgoyette  */
    454       1.1     skrll struct work_struct *
    455  1.1.24.1  pgoyette current_work(void)
    456       1.1     skrll {
    457  1.1.24.1  pgoyette 	struct workqueue_struct *wq = lwp_getspecific(workqueue_key);
    458  1.1.24.1  pgoyette 
    459  1.1.24.1  pgoyette 	/* If we're not a workqueue thread, then there's no work.  */
    460  1.1.24.1  pgoyette 	if (wq == NULL)
    461  1.1.24.1  pgoyette 		return NULL;
    462  1.1.24.1  pgoyette 
    463  1.1.24.1  pgoyette 	/*
    464  1.1.24.1  pgoyette 	 * Otherwise, this should be possible only while work is in
    465  1.1.24.1  pgoyette 	 * progress.  Return the current work item.
    466  1.1.24.1  pgoyette 	 */
    467       1.1     skrll 	KASSERT(wq->wq_current_work != NULL);
    468       1.1     skrll 	return wq->wq_current_work;
    469       1.1     skrll }
    470  1.1.24.1  pgoyette 
    471       1.1     skrll /*
    473  1.1.24.1  pgoyette  * Work
    474  1.1.24.1  pgoyette  */
    475  1.1.24.1  pgoyette 
    476  1.1.24.1  pgoyette /*
    477  1.1.24.1  pgoyette  * INIT_WORK(work, fn)
    478  1.1.24.1  pgoyette  *
    479  1.1.24.1  pgoyette  *	Initialize work for use with a workqueue to call fn in a worker
    480  1.1.24.1  pgoyette  *	thread.  There is no corresponding destruction operation.
    481       1.1     skrll  */
    482       1.1     skrll void
    483  1.1.24.1  pgoyette INIT_WORK(struct work_struct *work, void (*fn)(struct work_struct *))
    484  1.1.24.1  pgoyette {
    485       1.1     skrll 
    486       1.1     skrll 	work->work_owner = 0;
    487  1.1.24.1  pgoyette 	work->func = fn;
    488  1.1.24.1  pgoyette }
    489  1.1.24.1  pgoyette 
    490  1.1.24.1  pgoyette /*
    491  1.1.24.1  pgoyette  * work_claimed(work, wq)
    492  1.1.24.1  pgoyette  *
    493  1.1.24.1  pgoyette  *	True if work is currently claimed by a workqueue, meaning it is
    494  1.1.24.1  pgoyette  *	either on the queue or scheduled in a callout.  The workqueue
    495  1.1.24.1  pgoyette  *	must be wq, and caller must hold wq's lock.
    496       1.1     skrll  */
    497       1.1     skrll static bool
    498  1.1.24.1  pgoyette work_claimed(struct work_struct *work, struct workqueue_struct *wq)
    499  1.1.24.1  pgoyette {
    500       1.1     skrll 
    501  1.1.24.1  pgoyette 	KASSERT(work_queue(work) == wq);
    502       1.1     skrll 	KASSERT(mutex_owned(&wq->wq_lock));
    503       1.1     skrll 
    504  1.1.24.1  pgoyette 	return work->work_owner & 1;
    505  1.1.24.1  pgoyette }
    506  1.1.24.1  pgoyette 
    507  1.1.24.1  pgoyette /*
    508  1.1.24.1  pgoyette  * work_queue(work)
    509  1.1.24.1  pgoyette  *
    510  1.1.24.1  pgoyette  *	Return the last queue that work was queued on, or NULL if it
    511  1.1.24.1  pgoyette  *	was never queued.
    512       1.1     skrll  */
    513       1.1     skrll static struct workqueue_struct *
    514  1.1.24.1  pgoyette work_queue(struct work_struct *work)
    515       1.1     skrll {
    516       1.1     skrll 
    517  1.1.24.1  pgoyette 	return (struct workqueue_struct *)(work->work_owner & ~(uintptr_t)1);
    518  1.1.24.1  pgoyette }
    519  1.1.24.1  pgoyette 
    520  1.1.24.1  pgoyette /*
    521  1.1.24.1  pgoyette  * acquire_work(work, wq)
    522  1.1.24.1  pgoyette  *
    523  1.1.24.1  pgoyette  *	Try to claim work for wq.  If work is already claimed, it must
    524  1.1.24.1  pgoyette  *	be claimed by wq; return false.  If work is not already
    525  1.1.24.1  pgoyette  *	claimed, claim it, issue a memory barrier to match any prior
    526  1.1.24.1  pgoyette  *	release_work, and return true.
    527  1.1.24.1  pgoyette  *
    528  1.1.24.1  pgoyette  *	Caller must hold wq's lock.
    529       1.1     skrll  */
    530  1.1.24.1  pgoyette static bool
    531  1.1.24.1  pgoyette acquire_work(struct work_struct *work, struct workqueue_struct *wq)
    532  1.1.24.1  pgoyette {
    533  1.1.24.1  pgoyette 	uintptr_t owner0, owner;
    534  1.1.24.1  pgoyette 
    535  1.1.24.1  pgoyette 	KASSERT(mutex_owned(&wq->wq_lock));
    536  1.1.24.1  pgoyette 	KASSERT(((uintptr_t)wq & 1) == 0);
    537  1.1.24.1  pgoyette 
    538  1.1.24.1  pgoyette 	owner = (uintptr_t)wq | 1;
    539  1.1.24.1  pgoyette 	do {
    540  1.1.24.1  pgoyette 		owner0 = work->work_owner;
    541  1.1.24.1  pgoyette 		if (owner0 & 1) {
    542  1.1.24.1  pgoyette 			KASSERT((owner0 & ~(uintptr_t)1) == (uintptr_t)wq);
    543  1.1.24.1  pgoyette 			return false;
    544  1.1.24.1  pgoyette 		}
    545  1.1.24.1  pgoyette 		KASSERT(owner0 == (uintptr_t)NULL || owner0 == (uintptr_t)wq);
    546  1.1.24.1  pgoyette 	} while (atomic_cas_uintptr(&work->work_owner, owner0, owner) !=
    547  1.1.24.1  pgoyette 	    owner0);
    548  1.1.24.1  pgoyette 
    549  1.1.24.1  pgoyette 	KASSERT(work_queue(work) == wq);
    550       1.1     skrll 	membar_enter();
    551  1.1.24.1  pgoyette 	SDT_PROBE2(sdt, linux, work, acquire,  work, wq);
    552       1.1     skrll 	return true;
    553  1.1.24.1  pgoyette }
    554  1.1.24.1  pgoyette 
    555  1.1.24.1  pgoyette /*
    556  1.1.24.1  pgoyette  * release_work(work, wq)
    557  1.1.24.1  pgoyette  *
    558  1.1.24.1  pgoyette  *	Issue a memory barrier to match any subsequent acquire_work and
    559       1.1     skrll  *	dissociate work from wq.
    560  1.1.24.1  pgoyette  *
    561  1.1.24.1  pgoyette  *	Caller must hold wq's lock and work must be associated with wq.
    562       1.1     skrll  */
    563       1.1     skrll static void
    564  1.1.24.1  pgoyette release_work(struct work_struct *work, struct workqueue_struct *wq)
    565  1.1.24.1  pgoyette {
    566  1.1.24.1  pgoyette 
    567  1.1.24.1  pgoyette 	KASSERT(work_queue(work) == wq);
    568  1.1.24.1  pgoyette 	KASSERT(mutex_owned(&wq->wq_lock));
    569  1.1.24.1  pgoyette 
    570  1.1.24.1  pgoyette 	SDT_PROBE2(sdt, linux, work, release,  work, wq);
    571  1.1.24.1  pgoyette 	membar_exit();
    572  1.1.24.1  pgoyette 
    573  1.1.24.1  pgoyette 	/*
    574  1.1.24.1  pgoyette 	 * Non-interlocked r/m/w is safe here because nobody else can
    575  1.1.24.1  pgoyette 	 * write to this while the claimed bit is setand the workqueue
    576       1.1     skrll 	 * lock is held.
    577       1.1     skrll 	 */
    578  1.1.24.1  pgoyette 	work->work_owner &= ~(uintptr_t)1;
    579  1.1.24.1  pgoyette }
    580  1.1.24.1  pgoyette 
    581  1.1.24.1  pgoyette /*
    582  1.1.24.1  pgoyette  * schedule_work(work)
    583  1.1.24.1  pgoyette  *
    584  1.1.24.1  pgoyette  *	If work is not already queued on system_wq, queue it to be run
    585  1.1.24.1  pgoyette  *	by system_wq's worker thread when it next can.  True if it was
    586  1.1.24.1  pgoyette  *	newly queued, false if it was already queued.  If the work was
    587  1.1.24.1  pgoyette  *	already running, queue it to run again.
    588  1.1.24.1  pgoyette  *
    589       1.1     skrll  *	Caller must ensure work is not queued to run on a different
    590       1.1     skrll  *	workqueue.
    591       1.1     skrll  */
    592  1.1.24.1  pgoyette bool
    593       1.1     skrll schedule_work(struct work_struct *work)
    594       1.1     skrll {
    595       1.1     skrll 
    596  1.1.24.1  pgoyette 	return queue_work(system_wq, work);
    597  1.1.24.1  pgoyette }
    598  1.1.24.1  pgoyette 
    599  1.1.24.1  pgoyette /*
    600  1.1.24.1  pgoyette  * queue_work(wq, work)
    601  1.1.24.1  pgoyette  *
    602  1.1.24.1  pgoyette  *	If work is not already queued on wq, queue it to be run by wq's
    603  1.1.24.1  pgoyette  *	worker thread when it next can.  True if it was newly queued,
    604  1.1.24.1  pgoyette  *	false if it was already queued.  If the work was already
    605  1.1.24.1  pgoyette  *	running, queue it to run again.
    606  1.1.24.1  pgoyette  *
    607       1.1     skrll  *	Caller must ensure work is not queued to run on a different
    608       1.1     skrll  *	workqueue.
    609       1.1     skrll  */
    610       1.1     skrll bool
    611       1.1     skrll queue_work(struct workqueue_struct *wq, struct work_struct *work)
    612       1.1     skrll {
    613       1.1     skrll 	bool newly_queued;
    614  1.1.24.1  pgoyette 
    615  1.1.24.1  pgoyette 	KASSERT(wq != NULL);
    616  1.1.24.1  pgoyette 
    617  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    618  1.1.24.1  pgoyette 	if (__predict_true(acquire_work(work, wq))) {
    619  1.1.24.1  pgoyette 		/*
    620  1.1.24.1  pgoyette 		 * It wasn't on any workqueue at all.  Put it on this
    621  1.1.24.1  pgoyette 		 * one, and signal the worker thread that there is work
    622  1.1.24.1  pgoyette 		 * to do.
    623  1.1.24.1  pgoyette 		 */
    624       1.1     skrll 		TAILQ_INSERT_TAIL(&wq->wq_queue, work, work_entry);
    625  1.1.24.1  pgoyette 		cv_broadcast(&wq->wq_cv);
    626  1.1.24.1  pgoyette 		SDT_PROBE2(sdt, linux, work, queue,  work, wq);
    627  1.1.24.1  pgoyette 		newly_queued = true;
    628  1.1.24.1  pgoyette 	} else {
    629  1.1.24.1  pgoyette 		/*
    630       1.1     skrll 		 * It was already on this workqueue.  Nothing to do
    631       1.1     skrll 		 * since it is already queued.
    632  1.1.24.1  pgoyette 		 */
    633       1.1     skrll 		newly_queued = false;
    634       1.1     skrll 	}
    635       1.1     skrll 	mutex_exit(&wq->wq_lock);
    636       1.1     skrll 
    637  1.1.24.1  pgoyette 	return newly_queued;
    638  1.1.24.1  pgoyette }
    639  1.1.24.1  pgoyette 
    640  1.1.24.1  pgoyette /*
    641  1.1.24.1  pgoyette  * cancel_work(work)
    642  1.1.24.1  pgoyette  *
    643  1.1.24.1  pgoyette  *	If work was queued, remove it from the queue and return true.
    644       1.1     skrll  *	If work was not queued, return false.  Work may still be
    645  1.1.24.1  pgoyette  *	running when this returns.
    646       1.1     skrll  */
    647  1.1.24.1  pgoyette bool
    648       1.1     skrll cancel_work(struct work_struct *work)
    649       1.1     skrll {
    650  1.1.24.1  pgoyette 	struct workqueue_struct *wq;
    651  1.1.24.1  pgoyette 	bool cancelled_p = false;
    652  1.1.24.1  pgoyette 
    653       1.1     skrll 	/* If there's no workqueue, nothing to cancel.   */
    654  1.1.24.1  pgoyette 	if ((wq = work_queue(work)) == NULL)
    655  1.1.24.1  pgoyette 		goto out;
    656  1.1.24.1  pgoyette 
    657  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    658  1.1.24.1  pgoyette 	if (__predict_false(work_queue(work) != wq)) {
    659  1.1.24.1  pgoyette 		/*
    660  1.1.24.1  pgoyette 		 * It has finished execution or been cancelled by
    661  1.1.24.1  pgoyette 		 * another thread, and has been moved off the
    662  1.1.24.1  pgoyette 		 * workqueue, so it's too to cancel.
    663  1.1.24.1  pgoyette 		 */
    664  1.1.24.1  pgoyette 		cancelled_p = false;
    665  1.1.24.1  pgoyette 	} else {
    666  1.1.24.1  pgoyette 		/* Check whether it's on the queue.  */
    667  1.1.24.1  pgoyette 		if (work_claimed(work, wq)) {
    668  1.1.24.1  pgoyette 			/*
    669  1.1.24.1  pgoyette 			 * It is still on the queue.  Take it off the
    670  1.1.24.1  pgoyette 			 * queue and report successful cancellation.
    671  1.1.24.1  pgoyette 			 */
    672  1.1.24.1  pgoyette 			TAILQ_REMOVE(&wq->wq_queue, work, work_entry);
    673  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, cancel,  work, wq);
    674  1.1.24.1  pgoyette 			release_work(work, wq);
    675  1.1.24.1  pgoyette 			/* Can't dereference work after this point.  */
    676  1.1.24.1  pgoyette 			cancelled_p = true;
    677  1.1.24.1  pgoyette 		} else {
    678  1.1.24.1  pgoyette 			/* Not on the queue.  Couldn't cancel it.  */
    679  1.1.24.1  pgoyette 			cancelled_p = false;
    680       1.1     skrll 		}
    681  1.1.24.1  pgoyette 	}
    682  1.1.24.1  pgoyette 	mutex_exit(&wq->wq_lock);
    683       1.1     skrll 
    684  1.1.24.1  pgoyette out:	return cancelled_p;
    685  1.1.24.1  pgoyette }
    686  1.1.24.1  pgoyette 
    687  1.1.24.1  pgoyette /*
    688  1.1.24.1  pgoyette  * cancel_work_sync(work)
    689  1.1.24.1  pgoyette  *
    690  1.1.24.1  pgoyette  *	If work was queued, remove it from the queue and return true.
    691  1.1.24.1  pgoyette  *	If work was not queued, return false.  Either way, if work is
    692  1.1.24.1  pgoyette  *	currently running, wait for it to complete.
    693  1.1.24.1  pgoyette  *
    694  1.1.24.1  pgoyette  *	May sleep.
    695  1.1.24.1  pgoyette  */
    696  1.1.24.1  pgoyette bool
    697  1.1.24.1  pgoyette cancel_work_sync(struct work_struct *work)
    698       1.1     skrll {
    699  1.1.24.1  pgoyette 	struct workqueue_struct *wq;
    700  1.1.24.1  pgoyette 	bool cancelled_p = false;
    701  1.1.24.1  pgoyette 
    702       1.1     skrll 	/* If there's no workqueue, nothing to cancel.   */
    703  1.1.24.1  pgoyette 	if ((wq = work_queue(work)) == NULL)
    704  1.1.24.1  pgoyette 		goto out;
    705  1.1.24.1  pgoyette 
    706  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    707  1.1.24.1  pgoyette 	if (__predict_false(work_queue(work) != wq)) {
    708  1.1.24.1  pgoyette 		/*
    709  1.1.24.1  pgoyette 		 * It has finished execution or been cancelled by
    710  1.1.24.1  pgoyette 		 * another thread, and has been moved off the
    711  1.1.24.1  pgoyette 		 * workqueue, so it's too late to cancel.
    712  1.1.24.1  pgoyette 		 */
    713  1.1.24.1  pgoyette 		cancelled_p = false;
    714  1.1.24.1  pgoyette 	} else {
    715  1.1.24.1  pgoyette 		/* Check whether it's on the queue.  */
    716  1.1.24.1  pgoyette 		if (work_claimed(work, wq)) {
    717  1.1.24.1  pgoyette 			/*
    718  1.1.24.1  pgoyette 			 * It is still on the queue.  Take it off the
    719  1.1.24.1  pgoyette 			 * queue and report successful cancellation.
    720  1.1.24.1  pgoyette 			 */
    721  1.1.24.1  pgoyette 			TAILQ_REMOVE(&wq->wq_queue, work, work_entry);
    722  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, cancel,  work, wq);
    723  1.1.24.1  pgoyette 			release_work(work, wq);
    724  1.1.24.1  pgoyette 			/* Can't dereference work after this point.  */
    725  1.1.24.1  pgoyette 			cancelled_p = true;
    726  1.1.24.1  pgoyette 		} else {
    727  1.1.24.1  pgoyette 			/* Not on the queue.  Couldn't cancel it.  */
    728  1.1.24.1  pgoyette 			cancelled_p = false;
    729  1.1.24.1  pgoyette 		}
    730       1.1     skrll 		/* If it's still running, wait for it to complete.  */
    731  1.1.24.1  pgoyette 		if (wq->wq_current_work == work)
    732       1.1     skrll 			wait_for_current_work(work, wq);
    733  1.1.24.1  pgoyette 	}
    734       1.1     skrll 	mutex_exit(&wq->wq_lock);
    735       1.1     skrll 
    736  1.1.24.1  pgoyette out:	return cancelled_p;
    737  1.1.24.1  pgoyette }
    738  1.1.24.1  pgoyette 
    739  1.1.24.1  pgoyette /*
    740  1.1.24.1  pgoyette  * wait_for_current_work(work, wq)
    741  1.1.24.1  pgoyette  *
    742  1.1.24.1  pgoyette  *	wq must be currently executing work.  Wait for it to finish.
    743       1.1     skrll  *
    744  1.1.24.1  pgoyette  *	Does not dereference work.
    745       1.1     skrll  */
    746  1.1.24.1  pgoyette static void
    747       1.1     skrll wait_for_current_work(struct work_struct *work, struct workqueue_struct *wq)
    748  1.1.24.1  pgoyette {
    749  1.1.24.1  pgoyette 	uint64_t gen;
    750       1.1     skrll 
    751  1.1.24.1  pgoyette 	KASSERT(mutex_owned(&wq->wq_lock));
    752  1.1.24.1  pgoyette 	KASSERT(wq->wq_current_work == work);
    753  1.1.24.1  pgoyette 
    754       1.1     skrll 	/* Wait only one generation in case it gets requeued quickly.  */
    755       1.1     skrll 	SDT_PROBE2(sdt, linux, work, wait__start,  work, wq);
    756  1.1.24.1  pgoyette 	gen = wq->wq_gen;
    757  1.1.24.1  pgoyette 	do {
    758       1.1     skrll 		cv_wait(&wq->wq_cv, &wq->wq_lock);
    759  1.1.24.1  pgoyette 	} while (wq->wq_current_work == work && wq->wq_gen == gen);
    760  1.1.24.1  pgoyette 	SDT_PROBE2(sdt, linux, work, wait__done,  work, wq);
    761  1.1.24.1  pgoyette }
    762  1.1.24.1  pgoyette 
    763       1.1     skrll /*
    765  1.1.24.1  pgoyette  * Delayed work
    766  1.1.24.1  pgoyette  */
    767  1.1.24.1  pgoyette 
    768  1.1.24.1  pgoyette /*
    769  1.1.24.1  pgoyette  * INIT_DELAYED_WORK(dw, fn)
    770  1.1.24.1  pgoyette  *
    771  1.1.24.1  pgoyette  *	Initialize dw for use with a workqueue to call fn in a worker
    772  1.1.24.1  pgoyette  *	thread after a delay.  There is no corresponding destruction
    773       1.1     skrll  *	operation.
    774       1.1     skrll  */
    775  1.1.24.1  pgoyette void
    776  1.1.24.1  pgoyette INIT_DELAYED_WORK(struct delayed_work *dw, void (*fn)(struct work_struct *))
    777  1.1.24.1  pgoyette {
    778       1.1     skrll 
    779  1.1.24.1  pgoyette 	INIT_WORK(&dw->work, fn);
    780  1.1.24.1  pgoyette 	dw->dw_state = DELAYED_WORK_IDLE;
    781  1.1.24.1  pgoyette 	dw->dw_resched = -1;
    782  1.1.24.1  pgoyette 
    783  1.1.24.1  pgoyette 	/*
    784  1.1.24.1  pgoyette 	 * Defer callout_init until we are going to schedule the
    785       1.1     skrll 	 * callout, which can then callout_destroy it, because
    786       1.1     skrll 	 * otherwise since there's no DESTROY_DELAYED_WORK or anything
    787  1.1.24.1  pgoyette 	 * we have no opportunity to call callout_destroy.
    788  1.1.24.1  pgoyette 	 */
    789  1.1.24.1  pgoyette }
    790  1.1.24.1  pgoyette 
    791  1.1.24.1  pgoyette /*
    792  1.1.24.1  pgoyette  * schedule_delayed_work(dw, ticks)
    793  1.1.24.1  pgoyette  *
    794  1.1.24.1  pgoyette  *	If it is not currently scheduled, schedule dw to run after
    795  1.1.24.1  pgoyette  *	ticks on system_wq.  If currently executing and not already
    796  1.1.24.1  pgoyette  *	rescheduled, reschedule it.  True if it was newly scheduled,
    797  1.1.24.1  pgoyette  *	false if it was already scheduled.
    798  1.1.24.1  pgoyette  *
    799  1.1.24.1  pgoyette  *	If ticks == 0, queue it to run as soon as the worker can,
    800       1.1     skrll  *	without waiting for the next callout tick to run.
    801       1.1     skrll  */
    802  1.1.24.1  pgoyette bool
    803  1.1.24.1  pgoyette schedule_delayed_work(struct delayed_work *dw, unsigned long ticks)
    804       1.1     skrll {
    805  1.1.24.1  pgoyette 
    806  1.1.24.1  pgoyette 	return queue_delayed_work(system_wq, dw, ticks);
    807  1.1.24.1  pgoyette }
    808  1.1.24.1  pgoyette 
    809  1.1.24.1  pgoyette /*
    810  1.1.24.1  pgoyette  * dw_callout_init(wq, dw)
    811  1.1.24.1  pgoyette  *
    812  1.1.24.1  pgoyette  *	Initialize the callout of dw and transition to
    813  1.1.24.1  pgoyette  *	DELAYED_WORK_SCHEDULED.  Caller must use callout_schedule.
    814       1.1     skrll  */
    815  1.1.24.1  pgoyette static void
    816  1.1.24.1  pgoyette dw_callout_init(struct workqueue_struct *wq, struct delayed_work *dw)
    817  1.1.24.1  pgoyette {
    818       1.1     skrll 
    819  1.1.24.1  pgoyette 	KASSERT(mutex_owned(&wq->wq_lock));
    820  1.1.24.1  pgoyette 	KASSERT(work_queue(&dw->work) == wq);
    821  1.1.24.1  pgoyette 	KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
    822  1.1.24.1  pgoyette 
    823  1.1.24.1  pgoyette 	callout_init(&dw->dw_callout, CALLOUT_MPSAFE);
    824       1.1     skrll 	callout_setfunc(&dw->dw_callout, &linux_workqueue_timeout, dw);
    825  1.1.24.1  pgoyette 	TAILQ_INSERT_HEAD(&wq->wq_delayed, dw, dw_entry);
    826  1.1.24.1  pgoyette 	dw->dw_state = DELAYED_WORK_SCHEDULED;
    827  1.1.24.1  pgoyette }
    828  1.1.24.1  pgoyette 
    829  1.1.24.1  pgoyette /*
    830  1.1.24.1  pgoyette  * dw_callout_destroy(wq, dw)
    831  1.1.24.1  pgoyette  *
    832  1.1.24.1  pgoyette  *	Destroy the callout of dw and transition to DELAYED_WORK_IDLE.
    833       1.1     skrll  */
    834  1.1.24.1  pgoyette static void
    835  1.1.24.1  pgoyette dw_callout_destroy(struct workqueue_struct *wq, struct delayed_work *dw)
    836  1.1.24.1  pgoyette {
    837  1.1.24.1  pgoyette 
    838  1.1.24.1  pgoyette 	KASSERT(mutex_owned(&wq->wq_lock));
    839       1.1     skrll 	KASSERT(work_queue(&dw->work) == wq);
    840  1.1.24.1  pgoyette 	KASSERT(dw->dw_state == DELAYED_WORK_SCHEDULED ||
    841  1.1.24.1  pgoyette 	    dw->dw_state == DELAYED_WORK_RESCHEDULED ||
    842  1.1.24.1  pgoyette 	    dw->dw_state == DELAYED_WORK_CANCELLED);
    843  1.1.24.1  pgoyette 
    844       1.1     skrll 	TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
    845  1.1.24.1  pgoyette 	callout_destroy(&dw->dw_callout);
    846       1.1     skrll 	dw->dw_resched = -1;
    847  1.1.24.1  pgoyette 	dw->dw_state = DELAYED_WORK_IDLE;
    848  1.1.24.1  pgoyette }
    849  1.1.24.1  pgoyette 
    850  1.1.24.1  pgoyette /*
    851  1.1.24.1  pgoyette  * cancel_delayed_work_done(wq, dw)
    852       1.1     skrll  *
    853  1.1.24.1  pgoyette  *	Complete cancellation of a delayed work: transition from
    854  1.1.24.1  pgoyette  *	DELAYED_WORK_CANCELLED to DELAYED_WORK_IDLE and off the
    855       1.1     skrll  *	workqueue.  Caller must not dereference dw after this returns.
    856       1.1     skrll  */
    857  1.1.24.1  pgoyette static void
    858  1.1.24.1  pgoyette cancel_delayed_work_done(struct workqueue_struct *wq, struct delayed_work *dw)
    859  1.1.24.1  pgoyette {
    860  1.1.24.1  pgoyette 
    861  1.1.24.1  pgoyette 	KASSERT(mutex_owned(&wq->wq_lock));
    862  1.1.24.1  pgoyette 	KASSERT(work_queue(&dw->work) == wq);
    863  1.1.24.1  pgoyette 	KASSERT(dw->dw_state == DELAYED_WORK_CANCELLED);
    864       1.1     skrll 
    865       1.1     skrll 	dw_callout_destroy(wq, dw);
    866  1.1.24.1  pgoyette 	release_work(&dw->work, wq);
    867  1.1.24.1  pgoyette 	/* Can't dereference dw after this point.  */
    868  1.1.24.1  pgoyette }
    869  1.1.24.1  pgoyette 
    870  1.1.24.1  pgoyette /*
    871  1.1.24.1  pgoyette  * queue_delayed_work(wq, dw, ticks)
    872  1.1.24.1  pgoyette  *
    873  1.1.24.1  pgoyette  *	If it is not currently scheduled, schedule dw to run after
    874  1.1.24.1  pgoyette  *	ticks on wq.  If currently queued, remove it from the queue
    875  1.1.24.1  pgoyette  *	first.
    876       1.1     skrll  *
    877       1.1     skrll  *	If ticks == 0, queue it to run as soon as the worker can,
    878       1.1     skrll  *	without waiting for the next callout tick to run.
    879       1.1     skrll  */
    880       1.1     skrll bool
    881       1.1     skrll queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
    882  1.1.24.1  pgoyette     unsigned long ticks)
    883  1.1.24.1  pgoyette {
    884  1.1.24.1  pgoyette 	bool newly_queued;
    885  1.1.24.1  pgoyette 
    886  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    887  1.1.24.1  pgoyette 	if (__predict_true(acquire_work(&dw->work, wq))) {
    888  1.1.24.1  pgoyette 		/*
    889       1.1     skrll 		 * It wasn't on any workqueue at all.  Schedule it to
    890  1.1.24.1  pgoyette 		 * run on this one.
    891  1.1.24.1  pgoyette 		 */
    892  1.1.24.1  pgoyette 		KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
    893  1.1.24.1  pgoyette 		if (ticks == 0) {
    894       1.1     skrll 			TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work,
    895  1.1.24.1  pgoyette 			    work_entry);
    896  1.1.24.1  pgoyette 			cv_broadcast(&wq->wq_cv);
    897  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, queue,  &dw->work, wq);
    898  1.1.24.1  pgoyette 		} else {
    899  1.1.24.1  pgoyette 			/*
    900  1.1.24.1  pgoyette 			 * Initialize a callout and schedule to run
    901  1.1.24.1  pgoyette 			 * after a delay.
    902       1.1     skrll 			 */
    903       1.1     skrll 			dw_callout_init(wq, dw);
    904  1.1.24.1  pgoyette 			callout_schedule(&dw->dw_callout, MIN(INT_MAX, ticks));
    905  1.1.24.1  pgoyette 			SDT_PROBE3(sdt, linux, work, schedule,  dw, wq, ticks);
    906  1.1.24.1  pgoyette 		}
    907  1.1.24.1  pgoyette 		newly_queued = true;
    908  1.1.24.1  pgoyette 	} else {
    909  1.1.24.1  pgoyette 		/* It was already on this workqueue.  */
    910  1.1.24.1  pgoyette 		switch (dw->dw_state) {
    911  1.1.24.1  pgoyette 		case DELAYED_WORK_IDLE:
    912  1.1.24.1  pgoyette 		case DELAYED_WORK_SCHEDULED:
    913  1.1.24.1  pgoyette 		case DELAYED_WORK_RESCHEDULED:
    914  1.1.24.1  pgoyette 			/* On the queue or already scheduled.  Leave it.  */
    915  1.1.24.1  pgoyette 			newly_queued = false;
    916  1.1.24.1  pgoyette 			break;
    917  1.1.24.1  pgoyette 		case DELAYED_WORK_CANCELLED:
    918  1.1.24.1  pgoyette 			/*
    919  1.1.24.1  pgoyette 			 * Scheduled and the callout began, but it was
    920  1.1.24.1  pgoyette 			 * cancelled.  Reschedule it.
    921  1.1.24.1  pgoyette 			 */
    922  1.1.24.1  pgoyette 			if (ticks == 0) {
    923  1.1.24.1  pgoyette 				dw->dw_state = DELAYED_WORK_SCHEDULED;
    924  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, queue,
    925  1.1.24.1  pgoyette 				    &dw->work, wq);
    926  1.1.24.1  pgoyette 			} else {
    927  1.1.24.1  pgoyette 				dw->dw_state = DELAYED_WORK_RESCHEDULED;
    928  1.1.24.1  pgoyette 				dw->dw_resched = MIN(INT_MAX, ticks);
    929  1.1.24.1  pgoyette 				SDT_PROBE3(sdt, linux, work, schedule,
    930  1.1.24.1  pgoyette 				    dw, wq, ticks);
    931  1.1.24.1  pgoyette 			}
    932  1.1.24.1  pgoyette 			newly_queued = true;
    933  1.1.24.1  pgoyette 			break;
    934       1.1     skrll 		default:
    935  1.1.24.1  pgoyette 			panic("invalid delayed work state: %d",
    936       1.1     skrll 			    dw->dw_state);
    937       1.1     skrll 		}
    938       1.1     skrll 	}
    939       1.1     skrll 	mutex_exit(&wq->wq_lock);
    940  1.1.24.1  pgoyette 
    941  1.1.24.1  pgoyette 	return newly_queued;
    942  1.1.24.1  pgoyette }
    943  1.1.24.1  pgoyette 
    944  1.1.24.1  pgoyette /*
    945  1.1.24.1  pgoyette  * mod_delayed_work(wq, dw, ticks)
    946  1.1.24.1  pgoyette  *
    947  1.1.24.1  pgoyette  *	Schedule dw to run after ticks.  If scheduled or queued,
    948  1.1.24.1  pgoyette  *	reschedule.  If ticks == 0, run without delay.
    949       1.1     skrll  *
    950       1.1     skrll  *	True if it modified the timer of an already scheduled work,
    951       1.1     skrll  *	false if it newly scheduled the work.
    952       1.1     skrll  */
    953       1.1     skrll bool
    954       1.1     skrll mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
    955  1.1.24.1  pgoyette     unsigned long ticks)
    956  1.1.24.1  pgoyette {
    957  1.1.24.1  pgoyette 	bool timer_modified;
    958  1.1.24.1  pgoyette 
    959  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
    960  1.1.24.1  pgoyette 	if (acquire_work(&dw->work, wq)) {
    961  1.1.24.1  pgoyette 		/*
    962       1.1     skrll 		 * It wasn't on any workqueue at all.  Schedule it to
    963  1.1.24.1  pgoyette 		 * run on this one.
    964  1.1.24.1  pgoyette 		 */
    965  1.1.24.1  pgoyette 		KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
    966  1.1.24.1  pgoyette 		if (ticks == 0) {
    967  1.1.24.1  pgoyette 			/*
    968  1.1.24.1  pgoyette 			 * Run immediately: put it on the queue and
    969  1.1.24.1  pgoyette 			 * signal the worker thread.
    970  1.1.24.1  pgoyette 			 */
    971       1.1     skrll 			TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work,
    972  1.1.24.1  pgoyette 			    work_entry);
    973  1.1.24.1  pgoyette 			cv_broadcast(&wq->wq_cv);
    974  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, queue,  &dw->work, wq);
    975  1.1.24.1  pgoyette 		} else {
    976  1.1.24.1  pgoyette 			/*
    977  1.1.24.1  pgoyette 			 * Initialize a callout and schedule to run
    978  1.1.24.1  pgoyette 			 * after a delay.
    979       1.1     skrll 			 */
    980       1.1     skrll 			dw_callout_init(wq, dw);
    981  1.1.24.1  pgoyette 			callout_schedule(&dw->dw_callout, MIN(INT_MAX, ticks));
    982  1.1.24.1  pgoyette 			SDT_PROBE3(sdt, linux, work, schedule,  dw, wq, ticks);
    983  1.1.24.1  pgoyette 		}
    984  1.1.24.1  pgoyette 		timer_modified = false;
    985  1.1.24.1  pgoyette 	} else {
    986  1.1.24.1  pgoyette 		/* It was already on this workqueue.  */
    987  1.1.24.1  pgoyette 		switch (dw->dw_state) {
    988  1.1.24.1  pgoyette 		case DELAYED_WORK_IDLE:
    989  1.1.24.1  pgoyette 			/* On the queue.  */
    990  1.1.24.1  pgoyette 			if (ticks == 0) {
    991  1.1.24.1  pgoyette 				/* Leave it be.  */
    992  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, cancel,
    993  1.1.24.1  pgoyette 				    &dw->work, wq);
    994  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, queue,
    995  1.1.24.1  pgoyette 				    &dw->work, wq);
    996  1.1.24.1  pgoyette 			} else {
    997  1.1.24.1  pgoyette 				/* Remove from the queue and schedule.  */
    998  1.1.24.1  pgoyette 				TAILQ_REMOVE(&wq->wq_dqueue, &dw->work,
    999  1.1.24.1  pgoyette 				    work_entry);
   1000  1.1.24.1  pgoyette 				dw_callout_init(wq, dw);
   1001  1.1.24.1  pgoyette 				callout_schedule(&dw->dw_callout,
   1002  1.1.24.1  pgoyette 				    MIN(INT_MAX, ticks));
   1003  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, cancel,
   1004  1.1.24.1  pgoyette 				    &dw->work, wq);
   1005  1.1.24.1  pgoyette 				SDT_PROBE3(sdt, linux, work, schedule,
   1006  1.1.24.1  pgoyette 				    dw, wq, ticks);
   1007  1.1.24.1  pgoyette 			}
   1008  1.1.24.1  pgoyette 			timer_modified = true;
   1009  1.1.24.1  pgoyette 			break;
   1010  1.1.24.1  pgoyette 		case DELAYED_WORK_SCHEDULED:
   1011  1.1.24.1  pgoyette 			/*
   1012  1.1.24.1  pgoyette 			 * It is scheduled to run after a delay.  Try
   1013  1.1.24.1  pgoyette 			 * to stop it and reschedule it; if we can't,
   1014  1.1.24.1  pgoyette 			 * either reschedule it or cancel it to put it
   1015  1.1.24.1  pgoyette 			 * on the queue, and inform the callout.
   1016  1.1.24.1  pgoyette 			 */
   1017  1.1.24.1  pgoyette 			if (callout_stop(&dw->dw_callout)) {
   1018  1.1.24.1  pgoyette 				/* Can't stop, callout has begun.  */
   1019  1.1.24.1  pgoyette 				if (ticks == 0) {
   1020  1.1.24.1  pgoyette 					/*
   1021  1.1.24.1  pgoyette 					 * We don't actually need to do
   1022  1.1.24.1  pgoyette 					 * anything.  The callout will
   1023  1.1.24.1  pgoyette 					 * queue it as soon as it gets
   1024  1.1.24.1  pgoyette 					 * the lock.
   1025  1.1.24.1  pgoyette 					 */
   1026  1.1.24.1  pgoyette 					SDT_PROBE2(sdt, linux, work, cancel,
   1027  1.1.24.1  pgoyette 					    &dw->work, wq);
   1028  1.1.24.1  pgoyette 					SDT_PROBE2(sdt, linux, work, queue,
   1029  1.1.24.1  pgoyette 					    &dw->work, wq);
   1030  1.1.24.1  pgoyette 				} else {
   1031  1.1.24.1  pgoyette 					/* Ask the callout to reschedule.  */
   1032  1.1.24.1  pgoyette 					dw->dw_state = DELAYED_WORK_RESCHEDULED;
   1033  1.1.24.1  pgoyette 					dw->dw_resched = MIN(INT_MAX, ticks);
   1034  1.1.24.1  pgoyette 					SDT_PROBE2(sdt, linux, work, cancel,
   1035  1.1.24.1  pgoyette 					    &dw->work, wq);
   1036  1.1.24.1  pgoyette 					SDT_PROBE3(sdt, linux, work, schedule,
   1037  1.1.24.1  pgoyette 					    dw, wq, ticks);
   1038  1.1.24.1  pgoyette 				}
   1039  1.1.24.1  pgoyette 			} else {
   1040  1.1.24.1  pgoyette 				/* We stopped the callout before it began.  */
   1041  1.1.24.1  pgoyette 				if (ticks == 0) {
   1042  1.1.24.1  pgoyette 					/*
   1043  1.1.24.1  pgoyette 					 * Run immediately: destroy the
   1044  1.1.24.1  pgoyette 					 * callout, put it on the
   1045  1.1.24.1  pgoyette 					 * queue, and signal the worker
   1046  1.1.24.1  pgoyette 					 * thread.
   1047  1.1.24.1  pgoyette 					 */
   1048  1.1.24.1  pgoyette 					dw_callout_destroy(wq, dw);
   1049  1.1.24.1  pgoyette 					TAILQ_INSERT_TAIL(&wq->wq_dqueue,
   1050  1.1.24.1  pgoyette 					    &dw->work, work_entry);
   1051  1.1.24.1  pgoyette 					cv_broadcast(&wq->wq_cv);
   1052  1.1.24.1  pgoyette 					SDT_PROBE2(sdt, linux, work, cancel,
   1053  1.1.24.1  pgoyette 					    &dw->work, wq);
   1054  1.1.24.1  pgoyette 					SDT_PROBE2(sdt, linux, work, queue,
   1055  1.1.24.1  pgoyette 					    &dw->work, wq);
   1056  1.1.24.1  pgoyette 				} else {
   1057  1.1.24.1  pgoyette 					/*
   1058  1.1.24.1  pgoyette 					 * Reschedule the callout.  No
   1059  1.1.24.1  pgoyette 					 * state change.
   1060  1.1.24.1  pgoyette 					 */
   1061  1.1.24.1  pgoyette 					callout_schedule(&dw->dw_callout,
   1062  1.1.24.1  pgoyette 					    MIN(INT_MAX, ticks));
   1063  1.1.24.1  pgoyette 					SDT_PROBE2(sdt, linux, work, cancel,
   1064  1.1.24.1  pgoyette 					    &dw->work, wq);
   1065  1.1.24.1  pgoyette 					SDT_PROBE3(sdt, linux, work, schedule,
   1066  1.1.24.1  pgoyette 					    dw, wq, ticks);
   1067  1.1.24.1  pgoyette 				}
   1068  1.1.24.1  pgoyette 			}
   1069  1.1.24.1  pgoyette 			timer_modified = true;
   1070  1.1.24.1  pgoyette 			break;
   1071  1.1.24.1  pgoyette 		case DELAYED_WORK_RESCHEDULED:
   1072  1.1.24.1  pgoyette 			/*
   1073  1.1.24.1  pgoyette 			 * Someone rescheduled it after the callout
   1074  1.1.24.1  pgoyette 			 * started but before the poor thing even had a
   1075  1.1.24.1  pgoyette 			 * chance to acquire the lock.
   1076  1.1.24.1  pgoyette 			 */
   1077  1.1.24.1  pgoyette 			if (ticks == 0) {
   1078  1.1.24.1  pgoyette 				/*
   1079  1.1.24.1  pgoyette 				 * We can just switch back to
   1080  1.1.24.1  pgoyette 				 * DELAYED_WORK_SCHEDULED so that the
   1081  1.1.24.1  pgoyette 				 * callout will queue the work as soon
   1082  1.1.24.1  pgoyette 				 * as it gets the lock.
   1083  1.1.24.1  pgoyette 				 */
   1084  1.1.24.1  pgoyette 				dw->dw_state = DELAYED_WORK_SCHEDULED;
   1085  1.1.24.1  pgoyette 				dw->dw_resched = -1;
   1086  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, cancel,
   1087  1.1.24.1  pgoyette 				    &dw->work, wq);
   1088  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, queue,
   1089  1.1.24.1  pgoyette 				    &dw->work, wq);
   1090  1.1.24.1  pgoyette 			} else {
   1091  1.1.24.1  pgoyette 				/* Change the rescheduled time.  */
   1092  1.1.24.1  pgoyette 				dw->dw_resched = ticks;
   1093  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, cancel,
   1094  1.1.24.1  pgoyette 				    &dw->work, wq);
   1095  1.1.24.1  pgoyette 				SDT_PROBE3(sdt, linux, work, schedule,
   1096  1.1.24.1  pgoyette 				    dw, wq, ticks);
   1097  1.1.24.1  pgoyette 			}
   1098  1.1.24.1  pgoyette 			timer_modified = true;
   1099  1.1.24.1  pgoyette 			break;
   1100  1.1.24.1  pgoyette 		case DELAYED_WORK_CANCELLED:
   1101  1.1.24.1  pgoyette 			/*
   1102  1.1.24.1  pgoyette 			 * Someone cancelled it after the callout
   1103  1.1.24.1  pgoyette 			 * started but before the poor thing even had a
   1104  1.1.24.1  pgoyette 			 * chance to acquire the lock.
   1105  1.1.24.1  pgoyette 			 */
   1106  1.1.24.1  pgoyette 			if (ticks == 0) {
   1107  1.1.24.1  pgoyette 				/*
   1108  1.1.24.1  pgoyette 				 * We can just switch back to
   1109  1.1.24.1  pgoyette 				 * DELAYED_WORK_SCHEDULED so that the
   1110  1.1.24.1  pgoyette 				 * callout will queue the work as soon
   1111  1.1.24.1  pgoyette 				 * as it gets the lock.
   1112  1.1.24.1  pgoyette 				 */
   1113  1.1.24.1  pgoyette 				dw->dw_state = DELAYED_WORK_SCHEDULED;
   1114  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, queue,
   1115  1.1.24.1  pgoyette 				    &dw->work, wq);
   1116  1.1.24.1  pgoyette 			} else {
   1117  1.1.24.1  pgoyette 				/* Ask it to reschedule.  */
   1118  1.1.24.1  pgoyette 				dw->dw_state = DELAYED_WORK_RESCHEDULED;
   1119  1.1.24.1  pgoyette 				dw->dw_resched = MIN(INT_MAX, ticks);
   1120  1.1.24.1  pgoyette 				SDT_PROBE3(sdt, linux, work, schedule,
   1121  1.1.24.1  pgoyette 				    dw, wq, ticks);
   1122  1.1.24.1  pgoyette 			}
   1123  1.1.24.1  pgoyette 			timer_modified = false;
   1124       1.1     skrll 			break;
   1125  1.1.24.1  pgoyette 		default:
   1126       1.1     skrll 			panic("invalid delayed work state: %d", dw->dw_state);
   1127       1.1     skrll 		}
   1128       1.1     skrll 	}
   1129       1.1     skrll 	mutex_exit(&wq->wq_lock);
   1130  1.1.24.1  pgoyette 
   1131  1.1.24.1  pgoyette 	return timer_modified;
   1132  1.1.24.1  pgoyette }
   1133  1.1.24.1  pgoyette 
   1134  1.1.24.1  pgoyette /*
   1135  1.1.24.1  pgoyette  * cancel_delayed_work(dw)
   1136  1.1.24.1  pgoyette  *
   1137  1.1.24.1  pgoyette  *	If work was scheduled or queued, remove it from the schedule or
   1138  1.1.24.1  pgoyette  *	queue and return true.  If work was not scheduled or queued,
   1139  1.1.24.1  pgoyette  *	return false.  Note that work may already be running; if it
   1140       1.1     skrll  *	hasn't been rescheduled or requeued, then cancel_delayed_work
   1141       1.1     skrll  *	will return false, and either way, cancel_delayed_work will NOT
   1142       1.1     skrll  *	wait for the work to complete.
   1143  1.1.24.1  pgoyette  */
   1144  1.1.24.1  pgoyette bool
   1145       1.1     skrll cancel_delayed_work(struct delayed_work *dw)
   1146  1.1.24.1  pgoyette {
   1147  1.1.24.1  pgoyette 	struct workqueue_struct *wq;
   1148  1.1.24.1  pgoyette 	bool cancelled_p;
   1149       1.1     skrll 
   1150  1.1.24.1  pgoyette 	/* If there's no workqueue, nothing to cancel.   */
   1151  1.1.24.1  pgoyette 	if ((wq = work_queue(&dw->work)) == NULL)
   1152  1.1.24.1  pgoyette 		return false;
   1153  1.1.24.1  pgoyette 
   1154  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
   1155  1.1.24.1  pgoyette 	if (__predict_false(work_queue(&dw->work) != wq)) {
   1156  1.1.24.1  pgoyette 		cancelled_p = false;
   1157  1.1.24.1  pgoyette 	} else {
   1158  1.1.24.1  pgoyette 		switch (dw->dw_state) {
   1159  1.1.24.1  pgoyette 		case DELAYED_WORK_IDLE:
   1160  1.1.24.1  pgoyette 			/*
   1161  1.1.24.1  pgoyette 			 * It is either on the queue or already running
   1162  1.1.24.1  pgoyette 			 * or both.
   1163  1.1.24.1  pgoyette 			 */
   1164  1.1.24.1  pgoyette 			if (work_claimed(&dw->work, wq)) {
   1165  1.1.24.1  pgoyette 				/* On the queue.  Remove and release.  */
   1166  1.1.24.1  pgoyette 				TAILQ_REMOVE(&wq->wq_dqueue, &dw->work,
   1167  1.1.24.1  pgoyette 				    work_entry);
   1168  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, cancel,
   1169  1.1.24.1  pgoyette 				    &dw->work, wq);
   1170  1.1.24.1  pgoyette 				release_work(&dw->work, wq);
   1171  1.1.24.1  pgoyette 				/* Can't dereference dw after this point.  */
   1172  1.1.24.1  pgoyette 				cancelled_p = true;
   1173  1.1.24.1  pgoyette 			} else {
   1174  1.1.24.1  pgoyette 				/* Not on the queue, so didn't cancel.  */
   1175  1.1.24.1  pgoyette 				cancelled_p = false;
   1176  1.1.24.1  pgoyette 			}
   1177  1.1.24.1  pgoyette 			break;
   1178  1.1.24.1  pgoyette 		case DELAYED_WORK_SCHEDULED:
   1179  1.1.24.1  pgoyette 			/*
   1180  1.1.24.1  pgoyette 			 * If it is scheduled, mark it cancelled and
   1181  1.1.24.1  pgoyette 			 * try to stop the callout before it starts.
   1182  1.1.24.1  pgoyette 			 *
   1183  1.1.24.1  pgoyette 			 * If it's too late and the callout has already
   1184  1.1.24.1  pgoyette 			 * begun to execute, tough.
   1185  1.1.24.1  pgoyette 			 *
   1186  1.1.24.1  pgoyette 			 * If we stopped the callout before it started,
   1187  1.1.24.1  pgoyette 			 * however, then destroy the callout and
   1188  1.1.24.1  pgoyette 			 * dissociate it from the workqueue ourselves.
   1189  1.1.24.1  pgoyette 			 */
   1190  1.1.24.1  pgoyette 			dw->dw_state = DELAYED_WORK_CANCELLED;
   1191  1.1.24.1  pgoyette 			cancelled_p = true;
   1192  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, cancel,  &dw->work, wq);
   1193  1.1.24.1  pgoyette 			if (!callout_stop(&dw->dw_callout))
   1194  1.1.24.1  pgoyette 				cancel_delayed_work_done(wq, dw);
   1195  1.1.24.1  pgoyette 			break;
   1196  1.1.24.1  pgoyette 		case DELAYED_WORK_RESCHEDULED:
   1197  1.1.24.1  pgoyette 			/*
   1198  1.1.24.1  pgoyette 			 * If it is being rescheduled, the callout has
   1199  1.1.24.1  pgoyette 			 * already fired.  We must ask it to cancel.
   1200  1.1.24.1  pgoyette 			 */
   1201  1.1.24.1  pgoyette 			dw->dw_state = DELAYED_WORK_CANCELLED;
   1202  1.1.24.1  pgoyette 			dw->dw_resched = -1;
   1203  1.1.24.1  pgoyette 			cancelled_p = true;
   1204  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, cancel,  &dw->work, wq);
   1205  1.1.24.1  pgoyette 			break;
   1206  1.1.24.1  pgoyette 		case DELAYED_WORK_CANCELLED:
   1207  1.1.24.1  pgoyette 			/*
   1208  1.1.24.1  pgoyette 			 * If it is being cancelled, the callout has
   1209  1.1.24.1  pgoyette 			 * already fired.  There is nothing more for us
   1210  1.1.24.1  pgoyette 			 * to do.  Someone else claims credit for
   1211  1.1.24.1  pgoyette 			 * cancelling it.
   1212  1.1.24.1  pgoyette 			 */
   1213  1.1.24.1  pgoyette 			cancelled_p = false;
   1214  1.1.24.1  pgoyette 			break;
   1215       1.1     skrll 		default:
   1216  1.1.24.1  pgoyette 			panic("invalid delayed work state: %d",
   1217       1.1     skrll 			    dw->dw_state);
   1218       1.1     skrll 		}
   1219       1.1     skrll 	}
   1220       1.1     skrll 	mutex_exit(&wq->wq_lock);
   1221  1.1.24.1  pgoyette 
   1222  1.1.24.1  pgoyette 	return cancelled_p;
   1223  1.1.24.1  pgoyette }
   1224  1.1.24.1  pgoyette 
   1225  1.1.24.1  pgoyette /*
   1226  1.1.24.1  pgoyette  * cancel_delayed_work_sync(dw)
   1227  1.1.24.1  pgoyette  *
   1228  1.1.24.1  pgoyette  *	If work was scheduled or queued, remove it from the schedule or
   1229  1.1.24.1  pgoyette  *	queue and return true.  If work was not scheduled or queued,
   1230       1.1     skrll  *	return false.  Note that work may already be running; if it
   1231       1.1     skrll  *	hasn't been rescheduled or requeued, then cancel_delayed_work
   1232       1.1     skrll  *	will return false; either way, wait for it to complete.
   1233  1.1.24.1  pgoyette  */
   1234  1.1.24.1  pgoyette bool
   1235       1.1     skrll cancel_delayed_work_sync(struct delayed_work *dw)
   1236  1.1.24.1  pgoyette {
   1237  1.1.24.1  pgoyette 	struct workqueue_struct *wq;
   1238  1.1.24.1  pgoyette 	bool cancelled_p;
   1239       1.1     skrll 
   1240  1.1.24.1  pgoyette 	/* If there's no workqueue, nothing to cancel.  */
   1241  1.1.24.1  pgoyette 	if ((wq = work_queue(&dw->work)) == NULL)
   1242  1.1.24.1  pgoyette 		return false;
   1243  1.1.24.1  pgoyette 
   1244  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
   1245  1.1.24.1  pgoyette 	if (__predict_false(work_queue(&dw->work) != wq)) {
   1246  1.1.24.1  pgoyette 		cancelled_p = false;
   1247  1.1.24.1  pgoyette 	} else {
   1248  1.1.24.1  pgoyette 		switch (dw->dw_state) {
   1249  1.1.24.1  pgoyette 		case DELAYED_WORK_IDLE:
   1250  1.1.24.1  pgoyette 			/*
   1251  1.1.24.1  pgoyette 			 * It is either on the queue or already running
   1252  1.1.24.1  pgoyette 			 * or both.
   1253  1.1.24.1  pgoyette 			 */
   1254  1.1.24.1  pgoyette 			if (work_claimed(&dw->work, wq)) {
   1255  1.1.24.1  pgoyette 				/* On the queue.  Remove and release.  */
   1256  1.1.24.1  pgoyette 				TAILQ_REMOVE(&wq->wq_dqueue, &dw->work,
   1257  1.1.24.1  pgoyette 				    work_entry);
   1258  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, cancel,
   1259  1.1.24.1  pgoyette 				    &dw->work, wq);
   1260  1.1.24.1  pgoyette 				release_work(&dw->work, wq);
   1261  1.1.24.1  pgoyette 				/* Can't dereference dw after this point.  */
   1262  1.1.24.1  pgoyette 				cancelled_p = true;
   1263  1.1.24.1  pgoyette 			} else {
   1264  1.1.24.1  pgoyette 				/* Not on the queue, so didn't cancel. */
   1265  1.1.24.1  pgoyette 				cancelled_p = false;
   1266  1.1.24.1  pgoyette 			}
   1267  1.1.24.1  pgoyette 			/* If it's still running, wait for it to complete.  */
   1268  1.1.24.1  pgoyette 			if (wq->wq_current_work == &dw->work)
   1269  1.1.24.1  pgoyette 				wait_for_current_work(&dw->work, wq);
   1270  1.1.24.1  pgoyette 			break;
   1271  1.1.24.1  pgoyette 		case DELAYED_WORK_SCHEDULED:
   1272  1.1.24.1  pgoyette 			/*
   1273  1.1.24.1  pgoyette 			 * If it is scheduled, mark it cancelled and
   1274  1.1.24.1  pgoyette 			 * try to stop the callout before it starts.
   1275  1.1.24.1  pgoyette 			 *
   1276  1.1.24.1  pgoyette 			 * If it's too late and the callout has already
   1277  1.1.24.1  pgoyette 			 * begun to execute, we must wait for it to
   1278  1.1.24.1  pgoyette 			 * complete.  But we got in soon enough to ask
   1279  1.1.24.1  pgoyette 			 * the callout not to run, so we successfully
   1280  1.1.24.1  pgoyette 			 * cancelled it in that case.
   1281  1.1.24.1  pgoyette 			 *
   1282  1.1.24.1  pgoyette 			 * If we stopped the callout before it started,
   1283  1.1.24.1  pgoyette 			 * then we must destroy the callout and
   1284  1.1.24.1  pgoyette 			 * dissociate it from the workqueue ourselves.
   1285  1.1.24.1  pgoyette 			 */
   1286  1.1.24.1  pgoyette 			dw->dw_state = DELAYED_WORK_CANCELLED;
   1287  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, cancel,  &dw->work, wq);
   1288  1.1.24.1  pgoyette 			if (!callout_halt(&dw->dw_callout, &wq->wq_lock))
   1289  1.1.24.1  pgoyette 				cancel_delayed_work_done(wq, dw);
   1290  1.1.24.1  pgoyette 			cancelled_p = true;
   1291  1.1.24.1  pgoyette 			break;
   1292  1.1.24.1  pgoyette 		case DELAYED_WORK_RESCHEDULED:
   1293  1.1.24.1  pgoyette 			/*
   1294  1.1.24.1  pgoyette 			 * If it is being rescheduled, the callout has
   1295  1.1.24.1  pgoyette 			 * already fired.  We must ask it to cancel and
   1296  1.1.24.1  pgoyette 			 * wait for it to complete.
   1297  1.1.24.1  pgoyette 			 */
   1298  1.1.24.1  pgoyette 			dw->dw_state = DELAYED_WORK_CANCELLED;
   1299  1.1.24.1  pgoyette 			dw->dw_resched = -1;
   1300  1.1.24.1  pgoyette 			SDT_PROBE2(sdt, linux, work, cancel,  &dw->work, wq);
   1301  1.1.24.1  pgoyette 			(void)callout_halt(&dw->dw_callout, &wq->wq_lock);
   1302  1.1.24.1  pgoyette 			cancelled_p = true;
   1303  1.1.24.1  pgoyette 			break;
   1304  1.1.24.1  pgoyette 		case DELAYED_WORK_CANCELLED:
   1305  1.1.24.1  pgoyette 			/*
   1306  1.1.24.1  pgoyette 			 * If it is being cancelled, the callout has
   1307  1.1.24.1  pgoyette 			 * already fired.  We need only wait for it to
   1308  1.1.24.1  pgoyette 			 * complete.  Someone else, however, claims
   1309  1.1.24.1  pgoyette 			 * credit for cancelling it.
   1310  1.1.24.1  pgoyette 			 */
   1311  1.1.24.1  pgoyette 			(void)callout_halt(&dw->dw_callout, &wq->wq_lock);
   1312  1.1.24.1  pgoyette 			cancelled_p = false;
   1313  1.1.24.1  pgoyette 			break;
   1314       1.1     skrll 		default:
   1315  1.1.24.1  pgoyette 			panic("invalid delayed work state: %d",
   1316       1.1     skrll 			    dw->dw_state);
   1317       1.1     skrll 		}
   1318       1.1     skrll 	}
   1319  1.1.24.1  pgoyette 	mutex_exit(&wq->wq_lock);
   1320  1.1.24.1  pgoyette 
   1321  1.1.24.1  pgoyette 	return cancelled_p;
   1322  1.1.24.1  pgoyette }
   1323  1.1.24.1  pgoyette 
   1324  1.1.24.1  pgoyette /*
   1326  1.1.24.1  pgoyette  * Flush
   1327  1.1.24.1  pgoyette  */
   1328  1.1.24.1  pgoyette 
   1329  1.1.24.1  pgoyette /*
   1330  1.1.24.1  pgoyette  * flush_scheduled_work()
   1331  1.1.24.1  pgoyette  *
   1332  1.1.24.1  pgoyette  *	Wait for all work queued on system_wq to complete.  This does
   1333       1.1     skrll  *	not include delayed work.
   1334  1.1.24.1  pgoyette  */
   1335  1.1.24.1  pgoyette void
   1336  1.1.24.1  pgoyette flush_scheduled_work(void)
   1337  1.1.24.1  pgoyette {
   1338  1.1.24.1  pgoyette 
   1339  1.1.24.1  pgoyette 	flush_workqueue(system_wq);
   1340  1.1.24.1  pgoyette }
   1341  1.1.24.1  pgoyette 
   1342  1.1.24.1  pgoyette /*
   1343  1.1.24.1  pgoyette  * flush_workqueue_locked(wq)
   1344  1.1.24.1  pgoyette  *
   1345       1.1     skrll  *	Wait for all work queued on wq to complete.  This does not
   1346  1.1.24.1  pgoyette  *	include delayed work.
   1347       1.1     skrll  *
   1348  1.1.24.1  pgoyette  *	Caller must hold wq's lock.
   1349       1.1     skrll  */
   1350  1.1.24.1  pgoyette static void
   1351       1.1     skrll flush_workqueue_locked(struct workqueue_struct *wq)
   1352  1.1.24.1  pgoyette {
   1353  1.1.24.1  pgoyette 	uint64_t gen;
   1354       1.1     skrll 
   1355       1.1     skrll 	KASSERT(mutex_owned(&wq->wq_lock));
   1356  1.1.24.1  pgoyette 
   1357  1.1.24.1  pgoyette 	/* Get the current generation number.  */
   1358       1.1     skrll 	gen = wq->wq_gen;
   1359  1.1.24.1  pgoyette 
   1360  1.1.24.1  pgoyette 	/*
   1361       1.1     skrll 	 * If there's a batch of work in progress, we must wait for the
   1362  1.1.24.1  pgoyette 	 * worker thread to finish that batch.
   1363  1.1.24.1  pgoyette 	 */
   1364  1.1.24.1  pgoyette 	if (wq->wq_current_work != NULL)
   1365  1.1.24.1  pgoyette 		gen++;
   1366  1.1.24.1  pgoyette 
   1367  1.1.24.1  pgoyette 	/*
   1368  1.1.24.1  pgoyette 	 * If there's any work yet to be claimed from the queue by the
   1369       1.1     skrll 	 * worker thread, we must wait for it to finish one more batch
   1370  1.1.24.1  pgoyette 	 * too.
   1371  1.1.24.1  pgoyette 	 */
   1372  1.1.24.1  pgoyette 	if (!TAILQ_EMPTY(&wq->wq_queue) || !TAILQ_EMPTY(&wq->wq_dqueue))
   1373  1.1.24.1  pgoyette 		gen++;
   1374  1.1.24.1  pgoyette 
   1375       1.1     skrll 	/* Wait until the generation number has caught up.  */
   1376       1.1     skrll 	SDT_PROBE1(sdt, linux, work, flush__start,  wq);
   1377  1.1.24.1  pgoyette 	while (wq->wq_gen < gen)
   1378  1.1.24.1  pgoyette 		cv_wait(&wq->wq_cv, &wq->wq_lock);
   1379  1.1.24.1  pgoyette 	SDT_PROBE1(sdt, linux, work, flush__done,  wq);
   1380  1.1.24.1  pgoyette }
   1381  1.1.24.1  pgoyette 
   1382  1.1.24.1  pgoyette /*
   1383  1.1.24.1  pgoyette  * flush_workqueue(wq)
   1384  1.1.24.1  pgoyette  *
   1385  1.1.24.1  pgoyette  *	Wait for all work queued on wq to complete.  This does not
   1386  1.1.24.1  pgoyette  *	include delayed work.
   1387  1.1.24.1  pgoyette  */
   1388  1.1.24.1  pgoyette void
   1389  1.1.24.1  pgoyette flush_workqueue(struct workqueue_struct *wq)
   1390  1.1.24.1  pgoyette {
   1391  1.1.24.1  pgoyette 
   1392  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
   1393  1.1.24.1  pgoyette 	flush_workqueue_locked(wq);
   1394  1.1.24.1  pgoyette 	mutex_exit(&wq->wq_lock);
   1395  1.1.24.1  pgoyette }
   1396  1.1.24.1  pgoyette 
   1397  1.1.24.1  pgoyette /*
   1398  1.1.24.1  pgoyette  * flush_work(work)
   1399  1.1.24.1  pgoyette  *
   1400       1.1     skrll  *	If work is queued or currently executing, wait for it to
   1401       1.1     skrll  *	complete.
   1402       1.1     skrll  */
   1403  1.1.24.1  pgoyette void
   1404  1.1.24.1  pgoyette flush_work(struct work_struct *work)
   1405  1.1.24.1  pgoyette {
   1406       1.1     skrll 	struct workqueue_struct *wq;
   1407  1.1.24.1  pgoyette 
   1408       1.1     skrll 	/* If there's no workqueue, nothing to flush.  */
   1409       1.1     skrll 	if ((wq = work_queue(work)) == NULL)
   1410  1.1.24.1  pgoyette 		return;
   1411  1.1.24.1  pgoyette 
   1412  1.1.24.1  pgoyette 	flush_workqueue(wq);
   1413  1.1.24.1  pgoyette }
   1414  1.1.24.1  pgoyette 
   1415  1.1.24.1  pgoyette /*
   1416  1.1.24.1  pgoyette  * flush_delayed_work(dw)
   1417  1.1.24.1  pgoyette  *
   1418  1.1.24.1  pgoyette  *	If dw is scheduled to run after a delay, queue it immediately
   1419       1.1     skrll  *	instead.  Then, if dw is queued or currently executing, wait
   1420       1.1     skrll  *	for it to complete.
   1421       1.1     skrll  */
   1422  1.1.24.1  pgoyette void
   1423  1.1.24.1  pgoyette flush_delayed_work(struct delayed_work *dw)
   1424  1.1.24.1  pgoyette {
   1425       1.1     skrll 	struct workqueue_struct *wq;
   1426       1.1     skrll 
   1427  1.1.24.1  pgoyette 	/* If there's no workqueue, nothing to flush.  */
   1428  1.1.24.1  pgoyette 	if ((wq = work_queue(&dw->work)) == NULL)
   1429  1.1.24.1  pgoyette 		return;
   1430  1.1.24.1  pgoyette 
   1431  1.1.24.1  pgoyette 	mutex_enter(&wq->wq_lock);
   1432  1.1.24.1  pgoyette 	if (__predict_false(work_queue(&dw->work) != wq)) {
   1433       1.1     skrll 		/*
   1434  1.1.24.1  pgoyette 		 * Moved off the queue already (and possibly to another
   1435  1.1.24.1  pgoyette 		 * queue, though that would be ill-advised), so it must
   1436  1.1.24.1  pgoyette 		 * have completed, and we have nothing more to do.
   1437  1.1.24.1  pgoyette 		 */
   1438  1.1.24.1  pgoyette 	} else {
   1439  1.1.24.1  pgoyette 		switch (dw->dw_state) {
   1440  1.1.24.1  pgoyette 		case DELAYED_WORK_IDLE:
   1441  1.1.24.1  pgoyette 			/*
   1442  1.1.24.1  pgoyette 			 * It has a workqueue assigned and the callout
   1443  1.1.24.1  pgoyette 			 * is idle, so it must be in progress or on the
   1444  1.1.24.1  pgoyette 			 * queue.  In that case, we'll wait for it to
   1445  1.1.24.1  pgoyette 			 * complete.
   1446  1.1.24.1  pgoyette 			 */
   1447  1.1.24.1  pgoyette 			break;
   1448  1.1.24.1  pgoyette 		case DELAYED_WORK_SCHEDULED:
   1449  1.1.24.1  pgoyette 		case DELAYED_WORK_RESCHEDULED:
   1450  1.1.24.1  pgoyette 		case DELAYED_WORK_CANCELLED:
   1451  1.1.24.1  pgoyette 			/*
   1452  1.1.24.1  pgoyette 			 * The callout is scheduled, and may have even
   1453  1.1.24.1  pgoyette 			 * started.  Mark it as scheduled so that if
   1454  1.1.24.1  pgoyette 			 * the callout has fired it will queue the work
   1455  1.1.24.1  pgoyette 			 * itself.  Try to stop the callout -- if we
   1456  1.1.24.1  pgoyette 			 * can, queue the work now; if we can't, wait
   1457  1.1.24.1  pgoyette 			 * for the callout to complete, which entails
   1458  1.1.24.1  pgoyette 			 * queueing it.
   1459  1.1.24.1  pgoyette 			 */
   1460  1.1.24.1  pgoyette 			dw->dw_state = DELAYED_WORK_SCHEDULED;
   1461  1.1.24.1  pgoyette 			if (!callout_halt(&dw->dw_callout, &wq->wq_lock)) {
   1462  1.1.24.1  pgoyette 				/*
   1463  1.1.24.1  pgoyette 				 * We stopped it before it ran.  No
   1464  1.1.24.1  pgoyette 				 * state change in the interim is
   1465  1.1.24.1  pgoyette 				 * possible.  Destroy the callout and
   1466  1.1.24.1  pgoyette 				 * queue it ourselves.
   1467  1.1.24.1  pgoyette 				 */
   1468  1.1.24.1  pgoyette 				KASSERT(dw->dw_state ==
   1469  1.1.24.1  pgoyette 				    DELAYED_WORK_SCHEDULED);
   1470  1.1.24.1  pgoyette 				dw_callout_destroy(wq, dw);
   1471  1.1.24.1  pgoyette 				TAILQ_INSERT_TAIL(&wq->wq_dqueue, &dw->work,
   1472  1.1.24.1  pgoyette 				    work_entry);
   1473  1.1.24.1  pgoyette 				cv_broadcast(&wq->wq_cv);
   1474  1.1.24.1  pgoyette 				SDT_PROBE2(sdt, linux, work, queue,
   1475  1.1.24.1  pgoyette 				    &dw->work, wq);
   1476  1.1.24.1  pgoyette 			}
   1477  1.1.24.1  pgoyette 			break;
   1478  1.1.24.1  pgoyette 		default:
   1479  1.1.24.1  pgoyette 			panic("invalid delayed work state: %d", dw->dw_state);
   1480  1.1.24.1  pgoyette 		}
   1481       1.1     skrll 		/*
   1482       1.1     skrll 		 * Waiting for the whole queue to flush is overkill,
   1483       1.1     skrll 		 * but doesn't hurt.
   1484                     		 */
   1485                     		flush_workqueue_locked(wq);
   1486                     	}
   1487                     	mutex_exit(&wq->wq_lock);
   1488                     }
   1489