Home | History | Annotate | Line # | Download | only in linux
linux_work.c revision 1.17
      1   1.2  riastrad /*	$NetBSD: linux_work.c,v 1.17 2018/08/27 14:59:20 riastradh Exp $	*/
      2   1.1     skrll 
      3   1.1     skrll /*-
      4  1.12  riastrad  * Copyright (c) 2018 The NetBSD Foundation, Inc.
      5   1.1     skrll  * All rights reserved.
      6   1.1     skrll  *
      7   1.1     skrll  * This code is derived from software contributed to The NetBSD Foundation
      8   1.1     skrll  * by Taylor R. Campbell.
      9   1.1     skrll  *
     10   1.1     skrll  * Redistribution and use in source and binary forms, with or without
     11   1.1     skrll  * modification, are permitted provided that the following conditions
     12   1.1     skrll  * are met:
     13   1.1     skrll  * 1. Redistributions of source code must retain the above copyright
     14   1.1     skrll  *    notice, this list of conditions and the following disclaimer.
     15   1.1     skrll  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.1     skrll  *    notice, this list of conditions and the following disclaimer in the
     17   1.1     skrll  *    documentation and/or other materials provided with the distribution.
     18   1.1     skrll  *
     19   1.1     skrll  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20   1.1     skrll  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21   1.1     skrll  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22   1.1     skrll  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23   1.1     skrll  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24   1.1     skrll  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25   1.1     skrll  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26   1.1     skrll  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27   1.1     skrll  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28   1.1     skrll  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29   1.1     skrll  * POSSIBILITY OF SUCH DAMAGE.
     30   1.1     skrll  */
     31   1.1     skrll 
     32   1.1     skrll #include <sys/cdefs.h>
     33   1.2  riastrad __KERNEL_RCSID(0, "$NetBSD: linux_work.c,v 1.17 2018/08/27 14:59:20 riastradh Exp $");
     34   1.1     skrll 
     35   1.1     skrll #include <sys/types.h>
     36   1.1     skrll #include <sys/atomic.h>
     37   1.1     skrll #include <sys/callout.h>
     38   1.1     skrll #include <sys/condvar.h>
     39   1.1     skrll #include <sys/errno.h>
     40   1.1     skrll #include <sys/kmem.h>
     41  1.12  riastrad #include <sys/kthread.h>
     42  1.12  riastrad #include <sys/lwp.h>
     43   1.1     skrll #include <sys/mutex.h>
     44   1.1     skrll #include <sys/queue.h>
     45   1.1     skrll 
     46   1.1     skrll #include <linux/workqueue.h>
     47   1.1     skrll 
     48   1.1     skrll struct workqueue_struct {
     49   1.1     skrll 	kmutex_t			wq_lock;
     50   1.1     skrll 	kcondvar_t			wq_cv;
     51   1.1     skrll 	TAILQ_HEAD(, delayed_work)	wq_delayed;
     52  1.12  riastrad 	TAILQ_HEAD(, work_struct)	wq_queue;
     53   1.1     skrll 	struct work_struct		*wq_current_work;
     54  1.12  riastrad 	int				wq_flags;
     55  1.12  riastrad 	struct lwp			*wq_lwp;
     56  1.12  riastrad 	uint64_t			wq_gen;
     57  1.12  riastrad 	bool				wq_requeued:1;
     58  1.12  riastrad 	bool				wq_dying:1;
     59   1.1     skrll };
     60   1.1     skrll 
     61  1.12  riastrad static void __dead	linux_workqueue_thread(void *);
     62  1.12  riastrad static void		linux_workqueue_timeout(void *);
     63  1.17  riastrad static struct workqueue_struct *
     64  1.17  riastrad 			acquire_work(struct work_struct *,
     65  1.17  riastrad 			    struct workqueue_struct *);
     66  1.17  riastrad static void		release_work(struct work_struct *,
     67  1.17  riastrad 			    struct workqueue_struct *);
     68  1.12  riastrad static void		queue_delayed_work_anew(struct workqueue_struct *,
     69  1.12  riastrad 			    struct delayed_work *, unsigned long);
     70  1.12  riastrad 
     71  1.12  riastrad static specificdata_key_t workqueue_key __read_mostly;
     72  1.12  riastrad 
     73  1.12  riastrad struct workqueue_struct	*system_wq __read_mostly;
     74  1.12  riastrad struct workqueue_struct	*system_long_wq __read_mostly;
     75  1.12  riastrad struct workqueue_struct	*system_power_efficient_wq __read_mostly;
     76   1.3  riastrad 
     77   1.1     skrll int
     78   1.1     skrll linux_workqueue_init(void)
     79   1.1     skrll {
     80  1.12  riastrad 	int error;
     81   1.3  riastrad 
     82  1.12  riastrad 	error = lwp_specific_key_create(&workqueue_key, NULL);
     83  1.12  riastrad 	if (error)
     84  1.12  riastrad 		goto fail0;
     85   1.1     skrll 
     86   1.1     skrll 	system_wq = alloc_ordered_workqueue("lnxsyswq", 0);
     87  1.12  riastrad 	if (system_wq == NULL) {
     88  1.12  riastrad 		error = ENOMEM;
     89  1.12  riastrad 		goto fail1;
     90  1.12  riastrad 	}
     91   1.2  riastrad 
     92   1.2  riastrad 	system_long_wq = alloc_ordered_workqueue("lnxlngwq", 0);
     93  1.12  riastrad 	if (system_long_wq == NULL) {
     94  1.12  riastrad 		error = ENOMEM;
     95  1.12  riastrad 		goto fail2;
     96  1.12  riastrad 	}
     97   1.1     skrll 
     98   1.6  riastrad 	system_power_efficient_wq = alloc_ordered_workqueue("lnxpwrwq", 0);
     99  1.12  riastrad 	if (system_long_wq == NULL) {
    100  1.12  riastrad 		error = ENOMEM;
    101  1.12  riastrad 		goto fail3;
    102  1.12  riastrad 	}
    103   1.6  riastrad 
    104   1.1     skrll 	return 0;
    105   1.2  riastrad 
    106  1.12  riastrad fail4: __unused
    107   1.6  riastrad 	destroy_workqueue(system_power_efficient_wq);
    108  1.12  riastrad fail3:	destroy_workqueue(system_long_wq);
    109  1.12  riastrad fail2:	destroy_workqueue(system_wq);
    110  1.12  riastrad fail1:	lwp_specific_key_delete(workqueue_key);
    111  1.12  riastrad fail0:	KASSERT(error);
    112  1.12  riastrad 	return error;
    113   1.1     skrll }
    114   1.1     skrll 
    115   1.1     skrll void
    116   1.1     skrll linux_workqueue_fini(void)
    117   1.1     skrll {
    118   1.2  riastrad 
    119  1.12  riastrad 	destroy_workqueue(system_power_efficient_wq);
    120   1.2  riastrad 	destroy_workqueue(system_long_wq);
    121   1.1     skrll 	destroy_workqueue(system_wq);
    122  1.12  riastrad 	lwp_specific_key_delete(workqueue_key);
    123   1.1     skrll }
    124   1.1     skrll 
    125   1.1     skrll /*
    127   1.1     skrll  * Workqueues
    128   1.1     skrll  */
    129   1.1     skrll 
    130  1.12  riastrad struct workqueue_struct *
    131   1.1     skrll alloc_ordered_workqueue(const char *name, int flags)
    132   1.1     skrll {
    133   1.1     skrll 	struct workqueue_struct *wq;
    134   1.1     skrll 	int error;
    135  1.12  riastrad 
    136   1.1     skrll 	KASSERT(flags == 0);
    137   1.1     skrll 
    138   1.1     skrll 	wq = kmem_alloc(sizeof(*wq), KM_SLEEP);
    139  1.12  riastrad 
    140   1.1     skrll 	mutex_init(&wq->wq_lock, MUTEX_DEFAULT, IPL_NONE);
    141   1.1     skrll 	cv_init(&wq->wq_cv, name);
    142  1.12  riastrad 	TAILQ_INIT(&wq->wq_delayed);
    143   1.1     skrll 	TAILQ_INIT(&wq->wq_queue);
    144   1.1     skrll 	wq->wq_current_work = NULL;
    145  1.12  riastrad 
    146  1.12  riastrad 	error = kthread_create(PRI_NONE,
    147  1.12  riastrad 	    KTHREAD_MPSAFE|KTHREAD_TS|KTHREAD_MUSTJOIN, NULL,
    148  1.12  riastrad 	    &linux_workqueue_thread, wq, &wq->wq_lwp, "%s", name);
    149  1.12  riastrad 	if (error)
    150   1.3  riastrad 		goto fail0;
    151   1.1     skrll 
    152  1.12  riastrad 	return wq;
    153  1.12  riastrad 
    154  1.12  riastrad fail0:	KASSERT(TAILQ_EMPTY(&wq->wq_queue));
    155  1.12  riastrad 	KASSERT(TAILQ_EMPTY(&wq->wq_delayed));
    156  1.12  riastrad 	cv_destroy(&wq->wq_cv);
    157  1.12  riastrad 	mutex_destroy(&wq->wq_lock);
    158  1.12  riastrad 	kmem_free(wq, sizeof(*wq));
    159   1.1     skrll 	return NULL;
    160   1.1     skrll }
    161   1.1     skrll 
    162   1.1     skrll void
    163   1.1     skrll destroy_workqueue(struct workqueue_struct *wq)
    164   1.1     skrll {
    165   1.1     skrll 
    166  1.12  riastrad 	/*
    167  1.12  riastrad 	 * Cancel all delayed work.  We do this first because any
    168  1.12  riastrad 	 * delayed work that that has already timed out, which we can't
    169   1.1     skrll 	 * cancel, may have queued new work.
    170   1.1     skrll 	 */
    171  1.12  riastrad 	for (;;) {
    172   1.1     skrll 		struct delayed_work *dw = NULL;
    173   1.1     skrll 
    174  1.12  riastrad 		mutex_enter(&wq->wq_lock);
    175   1.1     skrll 		if (!TAILQ_EMPTY(&wq->wq_delayed)) {
    176  1.12  riastrad 			dw = TAILQ_FIRST(&wq->wq_delayed);
    177  1.12  riastrad 			if (!callout_halt(&dw->dw_callout, &wq->wq_lock))
    178   1.1     skrll 				TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
    179   1.1     skrll 		}
    180   1.1     skrll 		mutex_exit(&wq->wq_lock);
    181   1.1     skrll 
    182   1.1     skrll 		if (dw == NULL)
    183   1.1     skrll 			break;
    184   1.1     skrll 		cancel_delayed_work_sync(dw);
    185   1.1     skrll 	}
    186  1.12  riastrad 
    187  1.12  riastrad 	/* Tell the thread to exit.  */
    188  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    189  1.12  riastrad 	wq->wq_dying = true;
    190  1.12  riastrad 	cv_broadcast(&wq->wq_cv);
    191  1.12  riastrad 	mutex_exit(&wq->wq_lock);
    192  1.12  riastrad 
    193  1.12  riastrad 	/* Wait for it to exit.  */
    194  1.12  riastrad 	(void)kthread_join(wq->wq_lwp);
    195   1.1     skrll 
    196  1.12  riastrad 	KASSERT(wq->wq_current_work == NULL);
    197  1.12  riastrad 	KASSERT(TAILQ_EMPTY(&wq->wq_queue));
    198   1.1     skrll 	KASSERT(TAILQ_EMPTY(&wq->wq_delayed));
    199   1.1     skrll 	cv_destroy(&wq->wq_cv);
    200   1.1     skrll 	mutex_destroy(&wq->wq_lock);
    201   1.1     skrll 
    202   1.1     skrll 	kmem_free(wq, sizeof(*wq));
    203   1.1     skrll }
    204   1.1     skrll 
    205  1.12  riastrad /*
    207   1.1     skrll  * Work thread and callout
    208  1.12  riastrad  */
    209  1.12  riastrad 
    210   1.1     skrll static void __dead
    211  1.12  riastrad linux_workqueue_thread(void *cookie)
    212  1.12  riastrad {
    213   1.1     skrll 	struct workqueue_struct *const wq = cookie;
    214  1.12  riastrad 	TAILQ_HEAD(, work_struct) tmp;
    215   1.1     skrll 
    216  1.12  riastrad 	lwp_setspecific(workqueue_key, wq);
    217  1.12  riastrad 
    218  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    219  1.12  riastrad 	for (;;) {
    220  1.12  riastrad 		/* Wait until there's activity.  If we're dying, stop.  */
    221  1.12  riastrad 		while (TAILQ_EMPTY(&wq->wq_queue) && !wq->wq_dying)
    222  1.12  riastrad 			cv_wait(&wq->wq_cv, &wq->wq_lock);
    223   1.1     skrll 		if (wq->wq_dying)
    224  1.12  riastrad 			break;
    225  1.12  riastrad 
    226  1.12  riastrad 		/* Grab a batch of work off the queue.  */
    227  1.12  riastrad 		KASSERT(!TAILQ_EMPTY(&wq->wq_queue));
    228  1.12  riastrad 		TAILQ_INIT(&tmp);
    229  1.12  riastrad 		TAILQ_CONCAT(&tmp, &wq->wq_queue, work_entry);
    230  1.12  riastrad 
    231  1.12  riastrad 		/* Process each work item in the batch.  */
    232  1.12  riastrad 		while (!TAILQ_EMPTY(&tmp)) {
    233  1.12  riastrad 			struct work_struct *const work = TAILQ_FIRST(&tmp);
    234  1.12  riastrad 
    235  1.12  riastrad 			TAILQ_REMOVE(&tmp, work, work_entry);
    236   1.1     skrll 			KASSERT(wq->wq_current_work == NULL);
    237  1.12  riastrad 			wq->wq_current_work = work;
    238  1.12  riastrad 
    239  1.12  riastrad 			mutex_exit(&wq->wq_lock);
    240   1.1     skrll 			(*work->func)(work);
    241  1.12  riastrad 			mutex_enter(&wq->wq_lock);
    242  1.12  riastrad 
    243  1.12  riastrad 			KASSERT(wq->wq_current_work == work);
    244  1.12  riastrad 			KASSERT(work->work_queue == wq);
    245  1.12  riastrad 			if (wq->wq_requeued)
    246  1.17  riastrad 				wq->wq_requeued = false;
    247  1.12  riastrad 			else
    248  1.12  riastrad 				release_work(work, wq);
    249  1.12  riastrad 			wq->wq_current_work = NULL;
    250   1.1     skrll 			cv_broadcast(&wq->wq_cv);
    251  1.12  riastrad 		}
    252  1.12  riastrad 
    253  1.12  riastrad 		/* Notify flush that we've completed a batch of work.  */
    254   1.1     skrll 		wq->wq_gen++;
    255  1.12  riastrad 		cv_broadcast(&wq->wq_cv);
    256   1.1     skrll 	}
    257  1.12  riastrad 	mutex_exit(&wq->wq_lock);
    258   1.1     skrll 
    259   1.1     skrll 	kthread_exit(0);
    260   1.1     skrll }
    261  1.12  riastrad 
    262   1.1     skrll static void
    263  1.12  riastrad linux_workqueue_timeout(void *cookie)
    264  1.12  riastrad {
    265   1.1     skrll 	struct delayed_work *const dw = cookie;
    266  1.12  riastrad 	struct workqueue_struct *const wq = dw->work.work_queue;
    267  1.14  riastrad 
    268  1.12  riastrad 	KASSERT(wq != NULL);
    269  1.12  riastrad 
    270  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    271  1.12  riastrad 	switch (dw->dw_state) {
    272  1.12  riastrad 	case DELAYED_WORK_IDLE:
    273  1.12  riastrad 		panic("delayed work callout uninitialized: %p", dw);
    274  1.12  riastrad 	case DELAYED_WORK_SCHEDULED:
    275  1.12  riastrad 		dw->dw_state = DELAYED_WORK_IDLE;
    276  1.12  riastrad 		callout_destroy(&dw->dw_callout);
    277  1.12  riastrad 		TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
    278  1.12  riastrad 		TAILQ_INSERT_TAIL(&wq->wq_queue, &dw->work, work_entry);
    279  1.12  riastrad 		cv_broadcast(&wq->wq_cv);
    280  1.12  riastrad 		break;
    281  1.12  riastrad 	case DELAYED_WORK_RESCHEDULED:
    282  1.12  riastrad 		dw->dw_state = DELAYED_WORK_SCHEDULED;
    283  1.12  riastrad 		break;
    284  1.12  riastrad 	case DELAYED_WORK_CANCELLED:
    285  1.12  riastrad 		dw->dw_state = DELAYED_WORK_IDLE;
    286  1.17  riastrad 		callout_destroy(&dw->dw_callout);
    287  1.12  riastrad 		TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
    288  1.12  riastrad 		release_work(&dw->work, wq);
    289  1.12  riastrad 		break;
    290  1.12  riastrad 	default:
    291  1.15  riastrad 		panic("delayed work callout in bad state: %p", dw);
    292  1.15  riastrad 	}
    293  1.12  riastrad 	KASSERT(dw->dw_state == DELAYED_WORK_IDLE ||
    294   1.1     skrll 	    dw->dw_state == DELAYED_WORK_SCHEDULED);
    295   1.1     skrll 	mutex_exit(&wq->wq_lock);
    296  1.12  riastrad }
    297  1.12  riastrad 
    298   1.1     skrll struct work_struct *
    299  1.12  riastrad current_work(void)
    300   1.1     skrll {
    301  1.12  riastrad 	struct workqueue_struct *wq = lwp_getspecific(workqueue_key);
    302  1.12  riastrad 
    303  1.12  riastrad 	/* If we're not a workqueue thread, then there's no work.  */
    304   1.1     skrll 	if (wq == NULL)
    305  1.12  riastrad 		return NULL;
    306  1.12  riastrad 
    307  1.12  riastrad 	/*
    308  1.12  riastrad 	 * Otherwise, this should be possible only while work is in
    309  1.12  riastrad 	 * progress.  Return the current work item.
    310  1.12  riastrad 	 */
    311   1.1     skrll 	KASSERT(wq->wq_current_work != NULL);
    312   1.1     skrll 	return wq->wq_current_work;
    313   1.1     skrll }
    314   1.1     skrll 
    315   1.1     skrll /*
    317   1.1     skrll  * Work
    318   1.1     skrll  */
    319   1.1     skrll 
    320   1.1     skrll void
    321  1.12  riastrad INIT_WORK(struct work_struct *work, void (*fn)(struct work_struct *))
    322   1.4  riastrad {
    323   1.1     skrll 
    324   1.1     skrll 	work->work_queue = NULL;
    325  1.17  riastrad 	work->func = fn;
    326  1.17  riastrad }
    327  1.17  riastrad 
    328  1.17  riastrad static struct workqueue_struct *
    329  1.17  riastrad acquire_work(struct work_struct *work, struct workqueue_struct *wq)
    330  1.17  riastrad {
    331  1.17  riastrad 	struct workqueue_struct *wq0;
    332  1.17  riastrad 
    333  1.17  riastrad 	KASSERT(mutex_owned(&wq->wq_lock));
    334  1.17  riastrad 
    335  1.17  riastrad 	wq0 = atomic_cas_ptr(&work->work_queue, NULL, wq);
    336  1.17  riastrad 	if (wq0 == NULL) {
    337  1.17  riastrad 		membar_enter();
    338  1.17  riastrad 		KASSERT(work->work_queue == wq);
    339  1.17  riastrad 	}
    340  1.17  riastrad 
    341  1.17  riastrad 	return wq0;
    342  1.17  riastrad }
    343  1.17  riastrad 
    344  1.17  riastrad static void
    345  1.17  riastrad release_work(struct work_struct *work, struct workqueue_struct *wq)
    346  1.17  riastrad {
    347  1.17  riastrad 
    348  1.17  riastrad 	KASSERT(work->work_queue == wq);
    349  1.17  riastrad 	KASSERT(mutex_owned(&wq->wq_lock));
    350  1.17  riastrad 
    351  1.17  riastrad 	membar_exit();
    352   1.1     skrll 	work->work_queue = NULL;
    353   1.1     skrll }
    354   1.1     skrll 
    355  1.12  riastrad bool
    356   1.1     skrll schedule_work(struct work_struct *work)
    357   1.1     skrll {
    358   1.1     skrll 
    359   1.1     skrll 	return queue_work(system_wq, work);
    360   1.1     skrll }
    361   1.1     skrll 
    362  1.12  riastrad bool
    363   1.1     skrll queue_work(struct workqueue_struct *wq, struct work_struct *work)
    364   1.1     skrll {
    365   1.1     skrll 	struct workqueue_struct *wq0;
    366   1.1     skrll 	bool newly_queued;
    367  1.12  riastrad 
    368  1.17  riastrad 	KASSERT(wq != NULL);
    369  1.12  riastrad 
    370   1.1     skrll 	mutex_enter(&wq->wq_lock);
    371  1.12  riastrad 	if (__predict_true((wq0 = acquire_work(work, wq)) == NULL)) {
    372  1.12  riastrad 		TAILQ_INSERT_TAIL(&wq->wq_queue, work, work_entry);
    373   1.1     skrll 		newly_queued = true;
    374   1.1     skrll 	} else {
    375  1.12  riastrad 		KASSERT(wq0 == wq);
    376   1.1     skrll 		newly_queued = false;
    377   1.1     skrll 	}
    378   1.1     skrll 	mutex_exit(&wq->wq_lock);
    379   1.1     skrll 
    380   1.1     skrll 	return newly_queued;
    381  1.12  riastrad }
    382   1.1     skrll 
    383  1.12  riastrad bool
    384   1.1     skrll cancel_work(struct work_struct *work)
    385   1.1     skrll {
    386  1.13  riastrad 	struct workqueue_struct *wq;
    387  1.13  riastrad 	bool cancelled_p = false;
    388  1.13  riastrad 
    389  1.13  riastrad 	/* If there's no workqueue, nothing to cancel.   */
    390  1.12  riastrad 	if ((wq = work->work_queue) == NULL)
    391  1.12  riastrad 		goto out;
    392  1.12  riastrad 
    393  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    394  1.12  riastrad 	if (__predict_false(work->work_queue != wq)) {
    395  1.12  riastrad 		cancelled_p = false;
    396  1.12  riastrad 	} else if (wq->wq_current_work == work) {
    397   1.1     skrll 		cancelled_p = false;
    398   1.1     skrll 	} else {
    399  1.12  riastrad 		TAILQ_REMOVE(&wq->wq_queue, work, work_entry);
    400   1.1     skrll 		cancelled_p = true;
    401  1.13  riastrad 	}
    402   1.1     skrll 	mutex_exit(&wq->wq_lock);
    403   1.1     skrll 
    404  1.12  riastrad out:	return cancelled_p;
    405  1.12  riastrad }
    406   1.1     skrll 
    407   1.1     skrll bool
    408  1.12  riastrad cancel_work_sync(struct work_struct *work)
    409   1.1     skrll {
    410  1.13  riastrad 	struct workqueue_struct *wq;
    411  1.13  riastrad 	bool cancelled_p = false;
    412  1.13  riastrad 
    413  1.13  riastrad 	/* If there's no workqueue, nothing to cancel.   */
    414   1.1     skrll 	if ((wq = work->work_queue) == NULL)
    415  1.12  riastrad 		goto out;
    416  1.12  riastrad 
    417  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    418  1.12  riastrad 	if (__predict_false(work->work_queue != wq)) {
    419  1.12  riastrad 		cancelled_p = false;
    420  1.12  riastrad 	} else if (wq->wq_current_work == work) {
    421  1.12  riastrad 		do {
    422  1.12  riastrad 			cv_wait(&wq->wq_cv, &wq->wq_lock);
    423  1.12  riastrad 		} while (wq->wq_current_work == work);
    424  1.12  riastrad 		cancelled_p = false;
    425  1.12  riastrad 	} else {
    426   1.1     skrll 		TAILQ_REMOVE(&wq->wq_queue, work, work_entry);
    427   1.1     skrll 		cancelled_p = true;
    428  1.13  riastrad 	}
    429   1.1     skrll 	mutex_exit(&wq->wq_lock);
    430   1.1     skrll 
    431   1.1     skrll out:	return cancelled_p;
    432   1.1     skrll }
    433   1.1     skrll 
    434   1.1     skrll /*
    436   1.1     skrll  * Delayed work
    437   1.1     skrll  */
    438  1.12  riastrad 
    439   1.1     skrll void
    440  1.12  riastrad INIT_DELAYED_WORK(struct delayed_work *dw, void (*fn)(struct work_struct *))
    441  1.12  riastrad {
    442  1.12  riastrad 
    443  1.12  riastrad 	INIT_WORK(&dw->work, fn);
    444  1.12  riastrad 	dw->dw_state = DELAYED_WORK_IDLE;
    445  1.12  riastrad 
    446  1.12  riastrad 	/*
    447  1.12  riastrad 	 * Defer callout_init until we are going to schedule the
    448   1.1     skrll 	 * callout, which can then callout_destroy it, because
    449   1.1     skrll 	 * otherwise since there's no DESTROY_DELAYED_WORK or anything
    450   1.1     skrll 	 * we have no opportunity to call callout_destroy.
    451   1.1     skrll 	 */
    452   1.1     skrll }
    453  1.12  riastrad 
    454   1.1     skrll bool
    455   1.1     skrll schedule_delayed_work(struct delayed_work *dw, unsigned long ticks)
    456   1.1     skrll {
    457  1.12  riastrad 
    458  1.12  riastrad 	return queue_delayed_work(system_wq, dw, ticks);
    459   1.1     skrll }
    460   1.1     skrll 
    461   1.1     skrll static void
    462  1.12  riastrad queue_delayed_work_anew(struct workqueue_struct *wq, struct delayed_work *dw,
    463  1.12  riastrad     unsigned long ticks)
    464  1.12  riastrad {
    465  1.12  riastrad 
    466  1.12  riastrad 	KASSERT(mutex_owned(&wq->wq_lock));
    467  1.12  riastrad 	KASSERT(dw->work.work_queue == wq);
    468  1.12  riastrad 	KASSERT((dw->dw_state == DELAYED_WORK_IDLE) ||
    469  1.12  riastrad 	    (dw->dw_state == DELAYED_WORK_SCHEDULED));
    470  1.12  riastrad 
    471   1.1     skrll 	if (ticks == 0) {
    472  1.12  riastrad 		if (dw->dw_state == DELAYED_WORK_SCHEDULED) {
    473  1.12  riastrad 			callout_destroy(&dw->dw_callout);
    474  1.12  riastrad 			TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
    475  1.12  riastrad 		} else {
    476  1.12  riastrad 			KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
    477  1.12  riastrad 		}
    478   1.1     skrll 		TAILQ_INSERT_TAIL(&wq->wq_queue, &dw->work, work_entry);
    479  1.12  riastrad 		dw->dw_state = DELAYED_WORK_IDLE;
    480  1.12  riastrad 	} else {
    481   1.1     skrll 		if (dw->dw_state == DELAYED_WORK_IDLE) {
    482  1.12  riastrad 			callout_init(&dw->dw_callout, CALLOUT_MPSAFE);
    483  1.12  riastrad 			callout_reset(&dw->dw_callout, MIN(INT_MAX, ticks),
    484   1.1     skrll 			    &linux_workqueue_timeout, dw);
    485  1.12  riastrad 			TAILQ_INSERT_HEAD(&wq->wq_delayed, dw, dw_entry);
    486  1.12  riastrad 		} else {
    487  1.12  riastrad 			KASSERT(dw->dw_state == DELAYED_WORK_SCHEDULED);
    488   1.1     skrll 		}
    489  1.12  riastrad 		dw->dw_state = DELAYED_WORK_SCHEDULED;
    490  1.12  riastrad 	}
    491  1.12  riastrad }
    492  1.12  riastrad 
    493  1.12  riastrad bool
    494  1.12  riastrad queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
    495   1.1     skrll     unsigned long ticks)
    496  1.12  riastrad {
    497  1.17  riastrad 	struct workqueue_struct *wq0;
    498  1.12  riastrad 	bool newly_queued;
    499  1.12  riastrad 
    500  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    501  1.12  riastrad 	if (__predict_true((wq0 = acquire_work(&dw->work, wq)) == NULL)) {
    502  1.12  riastrad 		KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
    503   1.1     skrll 		queue_delayed_work_anew(wq, dw, ticks);
    504   1.1     skrll 		newly_queued = true;
    505  1.12  riastrad 	} else {
    506   1.1     skrll 		KASSERT(wq0 == wq);
    507   1.1     skrll 		newly_queued = false;
    508   1.1     skrll 	}
    509   1.1     skrll 	mutex_exit(&wq->wq_lock);
    510   1.1     skrll 
    511   1.1     skrll 	return newly_queued;
    512   1.1     skrll }
    513   1.1     skrll 
    514  1.12  riastrad bool
    515   1.1     skrll mod_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
    516   1.1     skrll     unsigned long ticks)
    517  1.12  riastrad {
    518  1.17  riastrad 	struct workqueue_struct *wq0;
    519  1.12  riastrad 	bool timer_modified;
    520  1.12  riastrad 
    521  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    522  1.12  riastrad 	if ((wq0 = acquire_work(&dw->work, wq)) == NULL) {
    523  1.12  riastrad 		KASSERT(dw->dw_state == DELAYED_WORK_IDLE);
    524  1.12  riastrad 		queue_delayed_work_anew(wq, dw, ticks);
    525  1.12  riastrad 		timer_modified = false;
    526  1.12  riastrad 	} else {
    527  1.12  riastrad 		KASSERT(wq0 == wq);
    528  1.12  riastrad 		switch (dw->dw_state) {
    529  1.12  riastrad 		case DELAYED_WORK_IDLE:
    530  1.12  riastrad 			if (wq->wq_current_work != &dw->work) {
    531  1.12  riastrad 				/* Work is queued, but hasn't started yet.  */
    532  1.12  riastrad 				TAILQ_REMOVE(&wq->wq_queue, &dw->work,
    533  1.12  riastrad 				    work_entry);
    534  1.12  riastrad 				queue_delayed_work_anew(wq, dw, ticks);
    535  1.12  riastrad 				timer_modified = true;
    536  1.12  riastrad 			} else {
    537  1.12  riastrad 				/*
    538  1.12  riastrad 				 * Too late.  Queue it anew.  If that
    539  1.12  riastrad 				 * would skip the callout because it's
    540  1.12  riastrad 				 * immediate, notify the workqueue.
    541  1.12  riastrad 				 */
    542  1.12  riastrad 				wq->wq_requeued = ticks == 0;
    543  1.12  riastrad 				queue_delayed_work_anew(wq, dw, ticks);
    544  1.12  riastrad 				timer_modified = false;
    545  1.12  riastrad 			}
    546  1.12  riastrad 			break;
    547  1.12  riastrad 		case DELAYED_WORK_SCHEDULED:
    548  1.12  riastrad 			if (callout_stop(&dw->dw_callout)) {
    549  1.12  riastrad 				/*
    550  1.12  riastrad 				 * Too late to stop, but we got in
    551  1.12  riastrad 				 * before the callout acquired the
    552  1.12  riastrad 				 * lock.  Reschedule it and tell it
    553  1.12  riastrad 				 * we've done so.
    554  1.12  riastrad 				 */
    555  1.12  riastrad 				dw->dw_state = DELAYED_WORK_RESCHEDULED;
    556  1.12  riastrad 				callout_schedule(&dw->dw_callout,
    557  1.12  riastrad 				    MIN(INT_MAX, ticks));
    558  1.12  riastrad 			} else {
    559  1.12  riastrad 				/* Stopped it.  Queue it anew.  */
    560  1.12  riastrad 				queue_delayed_work_anew(wq, dw, ticks);
    561  1.12  riastrad 			}
    562  1.12  riastrad 			timer_modified = true;
    563  1.12  riastrad 			break;
    564  1.12  riastrad 		case DELAYED_WORK_RESCHEDULED:
    565  1.12  riastrad 		case DELAYED_WORK_CANCELLED:
    566  1.12  riastrad 			/*
    567  1.12  riastrad 			 * Someone modified the timer _again_, or
    568  1.12  riastrad 			 * cancelled it, after the callout started but
    569  1.12  riastrad 			 * before the poor thing even had a chance to
    570  1.12  riastrad 			 * acquire the lock.  Just reschedule it once
    571  1.12  riastrad 			 * more.
    572  1.12  riastrad 			 */
    573  1.12  riastrad 			callout_schedule(&dw->dw_callout, MIN(INT_MAX, ticks));
    574  1.12  riastrad 			dw->dw_state = DELAYED_WORK_RESCHEDULED;
    575  1.12  riastrad 			timer_modified = true;
    576   1.1     skrll 			break;
    577   1.1     skrll 		default:
    578  1.12  riastrad 			panic("invalid delayed work state: %d",
    579   1.1     skrll 			    dw->dw_state);
    580   1.1     skrll 		}
    581   1.1     skrll 	}
    582   1.1     skrll 	mutex_exit(&wq->wq_lock);
    583   1.1     skrll 
    584   1.1     skrll 	return timer_modified;
    585   1.1     skrll }
    586  1.12  riastrad 
    587  1.12  riastrad bool
    588   1.1     skrll cancel_delayed_work(struct delayed_work *dw)
    589  1.14  riastrad {
    590  1.14  riastrad 	struct workqueue_struct *wq;
    591  1.14  riastrad 	bool cancelled_p;
    592  1.14  riastrad 
    593  1.12  riastrad 	/* If there's no workqueue, nothing to cancel.   */
    594  1.12  riastrad 	if ((wq = dw->work.work_queue) == NULL)
    595  1.12  riastrad 		return false;
    596  1.12  riastrad 
    597  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    598  1.12  riastrad 	if (__predict_false(dw->work.work_queue != wq)) {
    599  1.12  riastrad 		cancelled_p = false;
    600  1.12  riastrad 	} else {
    601  1.12  riastrad 		switch (dw->dw_state) {
    602  1.12  riastrad 		case DELAYED_WORK_IDLE:
    603  1.12  riastrad 			if (wq->wq_current_work == &dw->work) {
    604  1.12  riastrad 				/* Too late, it's already running.  */
    605  1.12  riastrad 				cancelled_p = false;
    606  1.12  riastrad 			} else {
    607  1.12  riastrad 				/* Got in before it started.  Remove it.  */
    608  1.12  riastrad 				TAILQ_REMOVE(&wq->wq_queue, &dw->work,
    609  1.12  riastrad 				    work_entry);
    610  1.12  riastrad 				cancelled_p = true;
    611  1.12  riastrad 			}
    612  1.12  riastrad 			break;
    613  1.12  riastrad 		case DELAYED_WORK_SCHEDULED:
    614  1.12  riastrad 		case DELAYED_WORK_RESCHEDULED:
    615  1.12  riastrad 		case DELAYED_WORK_CANCELLED:
    616  1.12  riastrad 			if (callout_stop(&dw->dw_callout)) {
    617  1.12  riastrad 				/*
    618  1.12  riastrad 				 * Too late to stop, but we got in
    619  1.12  riastrad 				 * before the callout acquired the
    620  1.12  riastrad 				 * lock.  Tell it to give up.
    621  1.12  riastrad 				 */
    622  1.12  riastrad 				dw->dw_state = DELAYED_WORK_CANCELLED;
    623  1.12  riastrad 			} else {
    624  1.12  riastrad 				/* Stopped it.  Kill it.  */
    625  1.12  riastrad 				callout_destroy(&dw->dw_callout);
    626  1.16  riastrad 				TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
    627  1.12  riastrad 				dw->dw_state = DELAYED_WORK_IDLE;
    628  1.12  riastrad 			}
    629  1.12  riastrad 			cancelled_p = true;
    630  1.12  riastrad 			break;
    631   1.1     skrll 		default:
    632  1.12  riastrad 			panic("invalid delayed work state: %d",
    633   1.1     skrll 			    dw->dw_state);
    634   1.1     skrll 		}
    635   1.1     skrll 	}
    636   1.1     skrll 	mutex_exit(&wq->wq_lock);
    637   1.1     skrll 
    638   1.1     skrll 	return cancelled_p;
    639   1.1     skrll }
    640  1.12  riastrad 
    641  1.12  riastrad bool
    642   1.1     skrll cancel_delayed_work_sync(struct delayed_work *dw)
    643  1.14  riastrad {
    644  1.14  riastrad 	struct workqueue_struct *wq;
    645  1.14  riastrad 	bool cancelled_p;
    646  1.14  riastrad 
    647  1.12  riastrad 	/* If there's no workqueue, nothing to cancel.   */
    648  1.12  riastrad 	if ((wq = dw->work.work_queue) == NULL)
    649  1.12  riastrad 		return false;
    650  1.12  riastrad 
    651  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    652  1.12  riastrad 	if (__predict_false(dw->work.work_queue != wq)) {
    653  1.12  riastrad 		cancelled_p = false;
    654  1.12  riastrad 	} else {
    655  1.12  riastrad retry:		switch (dw->dw_state) {
    656  1.12  riastrad 		case DELAYED_WORK_IDLE:
    657  1.12  riastrad 			if (wq->wq_current_work == &dw->work) {
    658  1.12  riastrad 				/* Too late, it's already running.  Wait.  */
    659  1.12  riastrad 				do {
    660  1.12  riastrad 					cv_wait(&wq->wq_cv, &wq->wq_lock);
    661  1.12  riastrad 				} while (wq->wq_current_work == &dw->work);
    662  1.12  riastrad 				cancelled_p = false;
    663  1.12  riastrad 			} else {
    664  1.12  riastrad 				/* Got in before it started.  Remove it.  */
    665  1.12  riastrad 				TAILQ_REMOVE(&wq->wq_queue, &dw->work,
    666  1.12  riastrad 				    work_entry);
    667  1.12  riastrad 				cancelled_p = true;
    668  1.12  riastrad 			}
    669  1.12  riastrad 			break;
    670  1.12  riastrad 		case DELAYED_WORK_SCHEDULED:
    671  1.12  riastrad 		case DELAYED_WORK_RESCHEDULED:
    672  1.12  riastrad 		case DELAYED_WORK_CANCELLED:
    673  1.12  riastrad 			/*
    674  1.12  riastrad 			 * If it has started, tell it to stop, and wait
    675  1.12  riastrad 			 * for it to complete.  We drop the lock, so by
    676  1.12  riastrad 			 * the time the callout has completed, we must
    677  1.12  riastrad 			 * review the state again.
    678  1.12  riastrad 			 */
    679  1.12  riastrad 			dw->dw_state = DELAYED_WORK_CANCELLED;
    680  1.12  riastrad 			callout_halt(&dw->dw_callout, &wq->wq_lock);
    681  1.12  riastrad 			goto retry;
    682   1.1     skrll 		default:
    683  1.12  riastrad 			panic("invalid delayed work state: %d",
    684   1.1     skrll 			    dw->dw_state);
    685   1.1     skrll 		}
    686   1.1     skrll 	}
    687  1.12  riastrad 	mutex_exit(&wq->wq_lock);
    688  1.12  riastrad 
    689  1.12  riastrad 	return cancelled_p;
    690  1.12  riastrad }
    691   1.1     skrll 
    692   1.5  riastrad /*
    694   1.5  riastrad  * Flush
    695   1.5  riastrad  */
    696  1.12  riastrad 
    697   1.5  riastrad void
    698   1.5  riastrad flush_scheduled_work(void)
    699  1.12  riastrad {
    700  1.12  riastrad 
    701   1.1     skrll 	flush_workqueue(system_wq);
    702  1.12  riastrad }
    703   1.1     skrll 
    704  1.12  riastrad void
    705  1.12  riastrad flush_workqueue(struct workqueue_struct *wq)
    706  1.12  riastrad {
    707  1.12  riastrad 	uint64_t gen;
    708  1.12  riastrad 
    709  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    710   1.1     skrll 	gen = wq->wq_gen;
    711   1.1     skrll 	do {
    712  1.12  riastrad 		cv_wait(&wq->wq_cv, &wq->wq_lock);
    713  1.12  riastrad 	} while (gen == wq->wq_gen);
    714   1.1     skrll 	mutex_exit(&wq->wq_lock);
    715  1.14  riastrad }
    716   1.1     skrll 
    717  1.14  riastrad bool
    718  1.14  riastrad flush_work(struct work_struct *work)
    719  1.12  riastrad {
    720   1.1     skrll 	struct workqueue_struct *wq;
    721  1.12  riastrad 
    722  1.12  riastrad 	/* If there's no workqueue, nothing to flush.  */
    723   1.1     skrll 	if ((wq = work->work_queue) == NULL)
    724   1.1     skrll 		return false;
    725  1.12  riastrad 
    726  1.12  riastrad 	flush_workqueue(wq);
    727   1.1     skrll 	return true;
    728  1.14  riastrad }
    729  1.12  riastrad 
    730   1.1     skrll bool
    731  1.14  riastrad flush_delayed_work(struct delayed_work *dw)
    732  1.14  riastrad {
    733  1.12  riastrad 	struct workqueue_struct *wq;
    734   1.1     skrll 	bool do_flush = false;
    735   1.1     skrll 
    736  1.12  riastrad 	/* If there's no workqueue, nothing to flush.  */
    737  1.12  riastrad 	if ((wq = dw->work.work_queue) == NULL)
    738   1.1     skrll 		return false;
    739  1.12  riastrad 
    740  1.12  riastrad 	mutex_enter(&wq->wq_lock);
    741  1.12  riastrad 	if (__predict_false(dw->work.work_queue != wq)) {
    742  1.12  riastrad 		do_flush = true;
    743  1.12  riastrad 	} else {
    744  1.12  riastrad retry:		switch (dw->dw_state) {
    745  1.12  riastrad 		case DELAYED_WORK_IDLE:
    746  1.12  riastrad 			if (wq->wq_current_work != &dw->work) {
    747  1.12  riastrad 				TAILQ_REMOVE(&wq->wq_queue, &dw->work,
    748  1.12  riastrad 				    work_entry);
    749  1.12  riastrad 			} else {
    750  1.12  riastrad 				do_flush = true;
    751  1.12  riastrad 			}
    752  1.12  riastrad 			break;
    753  1.12  riastrad 		case DELAYED_WORK_SCHEDULED:
    754  1.12  riastrad 		case DELAYED_WORK_RESCHEDULED:
    755  1.12  riastrad 		case DELAYED_WORK_CANCELLED:
    756  1.12  riastrad 			dw->dw_state = DELAYED_WORK_CANCELLED;
    757  1.12  riastrad 			callout_halt(&dw->dw_callout, &wq->wq_lock);
    758   1.1     skrll 			goto retry;
    759  1.12  riastrad 		default:
    760   1.1     skrll 			panic("invalid delayed work state: %d",
    761  1.12  riastrad 			    dw->dw_state);
    762  1.12  riastrad 		}
    763   1.1     skrll 	}
    764  1.12  riastrad 	mutex_exit(&wq->wq_lock);
    765   1.1     skrll 
    766                 	if (do_flush)
    767                 		flush_workqueue(wq);
    768                 
    769                 	return true;
    770                 }
    771