Module Name: src
Committed By: riastradh
Date: Mon Dec 30 04:50:12 UTC 2013
Modified Files:
src/sys/external/bsd/drm2/drm [riastradh-drm2]: drm_module.c
src/sys/external/bsd/drm2/include/linux [riastradh-drm2]: workqueue.h
src/sys/modules/drm2 [riastradh-drm2]: Makefile
Added Files:
src/sys/external/bsd/drm2/linux [riastradh-drm2]: linux_work.c
Log Message:
First (plausible) draft of Linux workqueue implementation rework.
Untested, but this looks better than what was there before, or any of
the drafts leading up to this which got torn out of the typewriter,
crumpled up, and crudely tossed in frustration toward the wastepaper
basket by my desk alongside the empty bottles of Jack Daniels that
fueled them, or something like that.
Can't use multiple CPUs per workqueue. That requires some explicit
management of per-CPU workqueue state, since NetBSD's workqueue(9)
doesn't provide that or cancellation or flushing. Oops.
To generate a diff of this commit:
cvs rdiff -u -r1.1.2.6 -r1.1.2.7 src/sys/external/bsd/drm2/drm/drm_module.c
cvs rdiff -u -r1.1.2.10 -r1.1.2.11 \
src/sys/external/bsd/drm2/include/linux/workqueue.h
cvs rdiff -u -r0 -r1.1.2.1 src/sys/external/bsd/drm2/linux/linux_work.c
cvs rdiff -u -r1.1.2.42 -r1.1.2.43 src/sys/modules/drm2/Makefile
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/external/bsd/drm2/drm/drm_module.c
diff -u src/sys/external/bsd/drm2/drm/drm_module.c:1.1.2.6 src/sys/external/bsd/drm2/drm/drm_module.c:1.1.2.7
--- src/sys/external/bsd/drm2/drm/drm_module.c:1.1.2.6 Sun Sep 8 15:26:24 2013
+++ src/sys/external/bsd/drm2/drm/drm_module.c Mon Dec 30 04:50:12 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: drm_module.c,v 1.1.2.6 2013/09/08 15:26:24 riastradh Exp $ */
+/* $NetBSD: drm_module.c,v 1.1.2.7 2013/12/30 04:50:12 riastradh Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: drm_module.c,v 1.1.2.6 2013/09/08 15:26:24 riastradh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: drm_module.c,v 1.1.2.7 2013/12/30 04:50:12 riastradh Exp $");
#include <sys/types.h>
#include <sys/device.h>
@@ -39,6 +39,7 @@ __KERNEL_RCSID(0, "$NetBSD: drm_module.c
#include <linux/highmem.h>
#include <linux/mutex.h>
+#include <linux/workqueue.h>
#include <drm/drmP.h>
@@ -75,12 +76,20 @@ drm2_modcmd(modcmd_t cmd, void *arg __un
" %d", error);
return error;
}
+ error = linux_workqueue_init();
+ if (error) {
+ aprint_error("drm: unable to initialize workqueues:"
+ " %d", error);
+ linux_kmap_fini();
+ return error;
+ }
#ifdef _MODULE
error = config_init_component(cfdriver_ioconf_drm,
cfattach_ioconf_drm, cfdata_ioconf_drm);
if (error) {
aprint_error("drm: unable to init component: %d\n",
error);
+ linux_workqueue_fini();
linux_kmap_fini();
return error;
}
@@ -91,6 +100,7 @@ drm2_modcmd(modcmd_t cmd, void *arg __un
error);
(void)config_fini_component(cfdriver_ioconf_drm,
cfattach_ioconf_drm, cfdata_ioconf_drm);
+ linux_workqueue_fini();
linux_kmap_fini();
return error;
}
@@ -108,6 +118,7 @@ drm2_modcmd(modcmd_t cmd, void *arg __un
/* XXX Now what? Reattach the devsw? */
return error;
#endif
+ linux_workqueue_fini();
linux_kmap_fini();
linux_mutex_destroy(&drm_global_mutex);
return 0;
Index: src/sys/external/bsd/drm2/include/linux/workqueue.h
diff -u src/sys/external/bsd/drm2/include/linux/workqueue.h:1.1.2.10 src/sys/external/bsd/drm2/include/linux/workqueue.h:1.1.2.11
--- src/sys/external/bsd/drm2/include/linux/workqueue.h:1.1.2.10 Sun Sep 8 16:40:36 2013
+++ src/sys/external/bsd/drm2/include/linux/workqueue.h Mon Dec 30 04:50:12 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: workqueue.h,v 1.1.2.10 2013/09/08 16:40:36 riastradh Exp $ */
+/* $NetBSD: workqueue.h,v 1.1.2.11 2013/12/30 04:50:12 riastradh Exp $ */
/*-
* Copyright (c) 2013 The NetBSD Foundation, Inc.
@@ -32,258 +32,80 @@
#ifndef _LINUX_WORKQUEUE_H_
#define _LINUX_WORKQUEUE_H_
+#include <sys/types.h>
#include <sys/callout.h>
-#include <sys/condvar.h>
-#include <sys/mutex.h>
+#include <sys/queue.h>
#include <sys/workqueue.h>
-#include <asm/bug.h>
#include <linux/kernel.h>
-/*
- * XXX This implementation is a load of bollocks -- callouts and
- * workqueues are expedient, but wrong, if for no reason other than
- * that there is no destroy operation.
- *
- * XXX The amount of code in here is absurd; it should be given a
- * proper source file.
- */
+#define INIT_DELAYED_WORK linux_INIT_DELAYED_WORK
+#define INIT_WORK linux_INIT_WORK
+#define alloc_ordered_workqueue linux_alloc_ordered_workqueue
+#define cancel_delayed_work linux_cancel_delayed_work
+#define cancel_delayed_work_sync linux_cancel_delayed_work_sync
+#define cancel_work linux_cancel_work
+#define cancel_work_sync linux_cancel_work_sync
+#define destroy_workqueue linux_destroy_workqueue
+#define flush_workqueue linux_flush_workqueue
+#define queue_delayed_work linux_queue_delayed_work
+#define queue_work linux_queue_work
+#define schedule_delayed_work linux_schedule_delayed_work
+#define schedule_work linux_schedule_work
+#define system_wq linux_system_wq
+#define to_delayed_work linux_to_delayed_work
+
+struct workqueue_struct;
struct work_struct {
- void (*w_fn)(struct work_struct *);
- struct workqueue *w_wq;
struct work w_wk;
- kmutex_t w_lock;
- kcondvar_t w_cv;
+ __cpu_simple_lock_t w_lock; /* XXX */
enum {
WORK_IDLE,
- WORK_QUEUED,
+ WORK_DELAYED,
+ WORK_PENDING,
+ WORK_INVOKED,
WORK_CANCELLED,
- WORK_INFLIGHT,
- WORK_REQUEUED,
+ WORK_DELAYED_CANCELLED,
} w_state;
+ struct workqueue_struct *w_wq;
+ void (*w_fn)(struct work_struct *);
};
-static void __unused
-linux_work_fn(struct work *wk __unused, void *arg)
-{
- struct work_struct *const work = arg;
-
- mutex_spin_enter(&work->w_lock);
- switch (work->w_state) {
- case WORK_IDLE:
- panic("work ran while idle: %p", work);
- break;
-
- case WORK_QUEUED:
- work->w_state = WORK_INFLIGHT;
- mutex_spin_exit(&work->w_lock);
- (*work->w_fn)(work);
- mutex_spin_enter(&work->w_lock);
- switch (work->w_state) {
- case WORK_IDLE:
- case WORK_QUEUED:
- panic("work hosed while in flight: %p", work);
- break;
-
- case WORK_INFLIGHT:
- case WORK_CANCELLED:
- work->w_state = WORK_IDLE;
- cv_broadcast(&work->w_cv);
- break;
-
- case WORK_REQUEUED:
- workqueue_enqueue(work->w_wq, &work->w_wk, NULL);
- work->w_state = WORK_QUEUED;
- break;
-
- default:
- panic("work %p in bad state: %d", work,
- (int)work->w_state);
- break;
- }
- break;
-
- case WORK_CANCELLED:
- work->w_state = WORK_IDLE;
- cv_broadcast(&work->w_cv);
- break;
-
- case WORK_INFLIGHT:
- panic("work already in flight: %p", work);
- break;
-
- case WORK_REQUEUED:
- panic("work requeued while not in flight: %p", work);
- break;
-
- default:
- panic("work %p in bad state: %d", work, (int)work->w_state);
- break;
- }
- mutex_spin_exit(&work->w_lock);
-}
-
-static inline void
-INIT_WORK(struct work_struct *work, void (*fn)(struct work_struct *))
-{
- int error;
-
- work->w_fn = fn;
- error = workqueue_create(&work->w_wq, "lnxworkq", &linux_work_fn,
- work, PRI_NONE, IPL_VM, WQ_MPSAFE);
- if (error)
- panic("workqueue creation failed: %d", error); /* XXX */
-
- mutex_init(&work->w_lock, MUTEX_DEFAULT, IPL_VM);
- cv_init(&work->w_cv, "linxwork");
- work->w_state = WORK_IDLE;
-}
-
-static inline void
-schedule_work(struct work_struct *work)
-{
-
- mutex_spin_enter(&work->w_lock);
- switch (work->w_state) {
- case WORK_IDLE:
- workqueue_enqueue(work->w_wq, &work->w_wk, NULL);
- work->w_state = WORK_QUEUED;
- break;
-
- case WORK_CANCELLED:
- break;
-
- case WORK_INFLIGHT:
- work->w_state = WORK_REQUEUED;
- break;
-
- case WORK_QUEUED:
- case WORK_REQUEUED:
- break;
-
- default:
- panic("work %p in bad state: %d", work, (int)work->w_state);
- break;
- }
- mutex_spin_exit(&work->w_lock);
-}
-
-/*
- * XXX This API can't possibly be right because there is no interlock.
- */
-static inline bool
-cancel_work_sync(struct work_struct *work)
-{
- bool was_pending = false;
-
- mutex_spin_enter(&work->w_lock);
-retry: switch (work->w_state) {
- case WORK_IDLE:
- break;
-
- case WORK_QUEUED:
- case WORK_INFLIGHT:
- case WORK_REQUEUED:
- work->w_state = WORK_CANCELLED;
- /* FALLTHROUGH */
- case WORK_CANCELLED:
- cv_wait(&work->w_cv, &work->w_lock);
- was_pending = true;
- goto retry;
-
- default:
- panic("work %p in bad state: %d", work, (int)work->w_state);
- }
- mutex_spin_exit(&work->w_lock);
-
- return was_pending;
-}
-
struct delayed_work {
- struct callout dw_callout;
- struct work_struct work; /* not dw_work; name must match Linux */
+ /* Not dw_work; name must match Linux. */
+ struct work_struct work;
+ struct callout dw_callout;
+ TAILQ_ENTRY(delayed_work) dw_entry;
};
-static void __unused
-linux_delayed_work_fn(void *arg)
-{
- struct delayed_work *const dw = arg;
-
- schedule_work(&dw->work);
-}
-
-static inline void
-INIT_DELAYED_WORK(struct delayed_work *dw, void (*fn)(struct work_struct *))
-{
- callout_init(&dw->dw_callout, CALLOUT_MPSAFE);
- callout_setfunc(&dw->dw_callout, linux_delayed_work_fn, dw);
- INIT_WORK(&dw->work, fn);
-}
-
static inline struct delayed_work *
to_delayed_work(struct work_struct *work)
{
return container_of(work, struct delayed_work, work);
}
-static inline void
-schedule_delayed_work(struct delayed_work *dw, unsigned long ticks)
-{
- KASSERT(ticks < INT_MAX);
- callout_schedule(&dw->dw_callout, (int)ticks);
-}
-
-static inline bool
-cancel_delayed_work(struct delayed_work *dw)
-{
- return !callout_stop(&dw->dw_callout);
-}
-
-static inline bool
-cancel_delayed_work_sync(struct delayed_work *dw)
-{
- const bool callout_was_pending = !callout_stop(&dw->dw_callout);
- const bool work_was_pending = cancel_work_sync(&dw->work);
-
- return (callout_was_pending || work_was_pending);
-}
-
-/*
- * XXX Bogus stubs for Linux work queues.
- */
-
-struct workqueue_struct;
-
-static inline struct workqueue_struct *
-alloc_ordered_workqueue(const char *name __unused, int flags __unused)
-{
- return (void *)(uintptr_t)0xdeadbeef;
-}
-
-static inline void
-destroy_workqueue(struct workqueue_struct *wq __unused)
-{
- KASSERT(wq == (void *)(uintptr_t)0xdeadbeef);
-}
-
-#define flush_workqueue(wq) WARN(true, "Can't flush workqueues!");
-#define flush_scheduled_work(wq) WARN(true, "Can't flush workqueues!");
+extern struct workqueue_struct *system_wq;
-static inline void
-queue_work(struct workqueue_struct *wq __unused, struct work_struct *work)
-{
- KASSERT(wq == (void *)(uintptr_t)0xdeadbeef);
- schedule_work(work);
-}
+int linux_workqueue_init(void);
+void linux_workqueue_fini(void);
-static inline void
-queue_delayed_work(struct workqueue_struct *wq __unused,
- struct delayed_work *dw,
- unsigned int ticks)
-{
- KASSERT(wq == (void *)(uintptr_t)0xdeadbeef);
- schedule_delayed_work(dw, ticks);
-}
+struct workqueue_struct *
+ alloc_ordered_workqueue(const char *, int);
+void destroy_workqueue(struct workqueue_struct *);
+void flush_workqueue(struct workqueue_struct *);
+
+void INIT_WORK(struct work_struct *, void (*)(struct work_struct *));
+void schedule_work(struct work_struct *);
+void queue_work(struct workqueue_struct *, struct work_struct *);
+bool cancel_work_sync(struct work_struct *);
+
+void INIT_DELAYED_WORK(struct delayed_work *,
+ void (*)(struct work_struct *));
+void schedule_delayed_work(struct delayed_work *, unsigned long);
+void queue_delayed_work(struct workqueue_struct *, struct delayed_work *,
+ unsigned long ticks);
+bool cancel_delayed_work(struct delayed_work *);
+bool cancel_delayed_work_sync(struct delayed_work *);
#endif /* _LINUX_WORKQUEUE_H_ */
Index: src/sys/modules/drm2/Makefile
diff -u src/sys/modules/drm2/Makefile:1.1.2.42 src/sys/modules/drm2/Makefile:1.1.2.43
--- src/sys/modules/drm2/Makefile:1.1.2.42 Sun Sep 8 16:38:51 2013
+++ src/sys/modules/drm2/Makefile Mon Dec 30 04:50:12 2013
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.1.2.42 2013/09/08 16:38:51 riastradh Exp $
+# $NetBSD: Makefile,v 1.1.2.43 2013/12/30 04:50:12 riastradh Exp $
.include "../Makefile.inc"
.include "Makefile.inc"
@@ -57,6 +57,7 @@ SRCS+= linux_kmap.c
SRCS+= linux_i2c.c
SRCS+= linux_idr.c
SRCS+= linux_list_sort.c
+SRCS+= linux_work.c
COPTS.drm_crtc.c+= -Wno-shadow
COPTS.drm_edid.c+= -Wno-shadow
Added files:
Index: src/sys/external/bsd/drm2/linux/linux_work.c
diff -u /dev/null src/sys/external/bsd/drm2/linux/linux_work.c:1.1.2.1
--- /dev/null Mon Dec 30 04:50:12 2013
+++ src/sys/external/bsd/drm2/linux/linux_work.c Mon Dec 30 04:50:12 2013
@@ -0,0 +1,714 @@
+/* $NetBSD: linux_work.c,v 1.1.2.1 2013/12/30 04:50:12 riastradh Exp $ */
+
+/*-
+ * Copyright (c) 2013 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: linux_work.c,v 1.1.2.1 2013/12/30 04:50:12 riastradh Exp $");
+
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/atomic.h>
+#include <sys/callout.h>
+#include <sys/condvar.h>
+#include <sys/errno.h>
+#include <sys/intr.h>
+#include <sys/kmem.h>
+#include <sys/mutex.h>
+#include <sys/queue.h>
+#include <sys/systm.h>
+#include <sys/workqueue.h>
+
+#include <machine/lock.h>
+
+#include <linux/workqueue.h>
+
+struct workqueue_struct {
+ struct workqueue *wq_workqueue;
+
+ /* XXX The following should all be per-CPU. */
+ kmutex_t wq_lock;
+
+ /*
+ * Condvar for when any state related to this workqueue
+ * changes. XXX Could split this into multiple condvars for
+ * different purposes, but whatever...
+ */
+ kcondvar_t wq_cv;
+
+ TAILQ_HEAD(, delayed_work) wq_delayed;
+ struct work_struct *wq_current_work;
+};
+
+static void linux_work_lock_init(struct work_struct *);
+static void linux_work_lock(struct work_struct *);
+static void linux_work_unlock(struct work_struct *);
+static bool linux_work_locked(struct work_struct *);
+
+static void linux_wq_barrier(struct work_struct *);
+
+static void linux_wait_for_cancelled_work(struct work_struct *);
+static void linux_wait_for_invoked_work(struct work_struct *);
+static void linux_worker(struct work *, void *);
+
+static void linux_cancel_delayed_work_callout(struct delayed_work *, bool);
+static void linux_wait_for_delayed_cancelled_work(struct delayed_work *);
+static void linux_worker_intr(void *);
+
+struct workqueue_struct *system_wq;
+
+int
+linux_workqueue_init(void)
+{
+
+ system_wq = alloc_ordered_workqueue("lnxsyswq", 0);
+ if (system_wq == NULL)
+ return ENOMEM;
+
+ return 0;
+}
+
+void
+linux_workqueue_fini(void)
+{
+ destroy_workqueue(system_wq);
+ system_wq = NULL;
+}
+
+/*
+ * Workqueues
+ */
+
+struct workqueue_struct *
+alloc_ordered_workqueue(const char *name, int linux_flags)
+{
+ struct workqueue_struct *wq;
+ int flags = WQ_MPSAFE;
+ int error;
+
+ KASSERT(linux_flags == 0);
+
+ error = workqueue_create(&wq->wq_workqueue, name, &linux_worker,
+ NULL, PRI_NONE, IPL_NONE, flags);
+ if (error)
+ return NULL;
+
+ mutex_init(&wq->wq_lock, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&wq->wq_cv, name);
+ TAILQ_INIT(&wq->wq_delayed);
+ wq->wq_current_work = NULL;
+
+ return wq;
+}
+
+void
+destroy_workqueue(struct workqueue_struct *wq)
+{
+
+ /*
+ * Cancel all delayed work.
+ */
+ for (;;) {
+ struct delayed_work *dw;
+
+ mutex_enter(&wq->wq_lock);
+ if (TAILQ_EMPTY(&wq->wq_delayed)) {
+ dw = NULL;
+ } else {
+ dw = TAILQ_FIRST(&wq->wq_delayed);
+ TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
+ }
+ mutex_exit(&wq->wq_lock);
+
+ if (dw == NULL)
+ break;
+
+ cancel_delayed_work_sync(dw);
+ }
+
+ /*
+ * workqueue_destroy empties the queue; we need not wait for
+ * completion explicitly. However, we can't destroy the
+ * condvar or mutex until this is done.
+ */
+ workqueue_destroy(wq->wq_workqueue);
+ KASSERT(wq->wq_current_work == NULL);
+ wq->wq_workqueue = NULL;
+
+ cv_destroy(&wq->wq_cv);
+ mutex_destroy(&wq->wq_lock);
+}
+
+/*
+ * Flush
+ *
+ * Note: This doesn't cancel or wait for delayed work. This seems to
+ * match what Linux does (or, doesn't do).
+ */
+
+struct wq_flush_work {
+ struct work_struct wqfw_work;
+ struct wq_flush *wqfw_flush;
+};
+
+struct wq_flush {
+ kmutex_t wqf_lock;
+ kcondvar_t wqf_cv;
+ unsigned int wqf_n;
+};
+
+void
+flush_workqueue(struct workqueue_struct *wq)
+{
+ static const struct wq_flush zero_wqf;
+ struct wq_flush wqf = zero_wqf;
+
+ mutex_init(&wqf.wqf_lock, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&wqf.wqf_cv, "lnxwflsh");
+
+ if (1) {
+ static const struct wq_flush_work zero_wqfw;
+ struct wq_flush_work wqfw = zero_wqfw;
+
+ wqf.wqf_n = 1;
+ INIT_WORK(&wqfw.wqfw_work, &linux_wq_barrier);
+ wqfw.wqfw_work.w_wq = wq;
+ wqfw.wqfw_work.w_state = WORK_PENDING;
+ workqueue_enqueue(wq->wq_workqueue, &wqfw.wqfw_work.w_wk,
+ NULL);
+ } else {
+ struct cpu_info *ci;
+ CPU_INFO_ITERATOR cii;
+ struct wq_flush_work *wqfw;
+
+ panic("per-CPU Linux workqueues don't work yet!");
+
+ wqf.wqf_n = 0;
+ for (CPU_INFO_FOREACH(cii, ci)) {
+ wqfw = kmem_zalloc(sizeof(*wqfw), KM_SLEEP);
+ mutex_enter(&wqf.wqf_lock);
+ wqf.wqf_n++;
+ mutex_exit(&wqf.wqf_lock);
+ INIT_WORK(&wqfw->wqfw_work, &linux_wq_barrier);
+ wqfw->wqfw_work.w_state = WORK_PENDING;
+ wqfw->wqfw_work.w_wq = wq;
+ workqueue_enqueue(wq->wq_workqueue,
+ &wqfw->wqfw_work.w_wk, ci);
+ }
+ }
+
+ mutex_enter(&wqf.wqf_lock);
+ while (0 < wqf.wqf_n)
+ cv_wait(&wqf.wqf_cv, &wqf.wqf_lock);
+ mutex_exit(&wqf.wqf_lock);
+
+ cv_destroy(&wqf.wqf_cv);
+ mutex_destroy(&wqf.wqf_lock);
+}
+
+static void
+linux_wq_barrier(struct work_struct *work)
+{
+ struct wq_flush_work *const wqfw = container_of(work,
+ struct wq_flush_work, wqfw_work);
+ struct wq_flush *const wqf = wqfw->wqfw_flush;
+
+ mutex_enter(&wqf->wqf_lock);
+ if (--wqf->wqf_n == 0)
+ cv_broadcast(&wqf->wqf_cv);
+ mutex_exit(&wqf->wqf_lock);
+
+ kmem_free(wqfw, sizeof(*wqfw));
+}
+
+/*
+ * Work locking
+ *
+ * We use __cpu_simple_lock(9) rather than mutex(9) because Linux code
+ * does not destroy work, so there is nowhere to call mutex_destroy.
+ */
+
+static void
+linux_work_lock_init(struct work_struct *work)
+{
+ __cpu_simple_lock_init(&work->w_lock);
+}
+
+static void
+linux_work_lock(struct work_struct *work)
+{
+ __cpu_simple_lock(&work->w_lock);
+}
+
+static void
+linux_work_unlock(struct work_struct *work)
+{
+ __cpu_simple_unlock(&work->w_lock);
+}
+
+static bool
+linux_work_locked(struct work_struct *work)
+{
+ return __SIMPLELOCK_LOCKED_P(&work->w_lock);
+}
+
+/*
+ * Work
+ */
+
+void
+INIT_WORK(struct work_struct *work, void (*fn)(struct work_struct *))
+{
+
+ linux_work_lock_init(work);
+ work->w_state = WORK_IDLE;
+ work->w_wq = NULL;
+ work->w_fn = fn;
+}
+
+void
+schedule_work(struct work_struct *work)
+{
+ queue_work(system_wq, work);
+}
+
+void
+queue_work(struct workqueue_struct *wq, struct work_struct *work)
+{
+
+ KASSERT(wq != NULL);
+
+ linux_work_lock(work);
+ switch (work->w_state) {
+ case WORK_IDLE:
+ case WORK_INVOKED:
+ work->w_state = WORK_PENDING;
+ work->w_wq = wq;
+ workqueue_enqueue(wq->wq_workqueue, &work->w_wk, NULL);
+ break;
+
+ case WORK_DELAYED:
+ panic("queue_work(delayed work %p)", work);
+ break;
+
+ case WORK_PENDING:
+ KASSERT(work->w_wq == wq);
+ break;
+
+ case WORK_CANCELLED:
+ break;
+
+ case WORK_DELAYED_CANCELLED:
+ panic("queue_work(delayed work %p)", work);
+ break;
+
+ default:
+ panic("work %p in bad state: %d", work, (int)work->w_state);
+ break;
+ }
+ linux_work_unlock(work);
+}
+
+bool
+cancel_work_sync(struct work_struct *work)
+{
+ bool cancelled_p = false;
+
+ linux_work_lock(work);
+ switch (work->w_state) {
+ case WORK_IDLE: /* Nothing to do. */
+ break;
+
+ case WORK_DELAYED:
+ panic("cancel_work_sync(delayed work %p)", work);
+ break;
+
+ case WORK_PENDING:
+ work->w_state = WORK_CANCELLED;
+ linux_wait_for_cancelled_work(work);
+ cancelled_p = true;
+ break;
+
+ case WORK_INVOKED:
+ linux_wait_for_invoked_work(work);
+ break;
+
+ case WORK_CANCELLED: /* Already done. */
+ break;
+
+ case WORK_DELAYED_CANCELLED:
+ panic("cancel_work_sync(delayed work %p)", work);
+ break;
+
+ default:
+ panic("work %p in bad state: %d", work, (int)work->w_state);
+ break;
+ }
+ linux_work_unlock(work);
+
+ return cancelled_p;
+}
+
+static void
+linux_wait_for_cancelled_work(struct work_struct *work)
+{
+ struct workqueue_struct *wq;
+
+ KASSERT(linux_work_locked(work));
+ KASSERT(work->w_state == WORK_CANCELLED);
+
+ wq = work->w_wq;
+ do {
+ mutex_enter(&wq->wq_lock);
+ linux_work_unlock(work);
+ cv_wait(&wq->wq_cv, &wq->wq_lock);
+ mutex_exit(&wq->wq_lock);
+ linux_work_lock(work);
+ } while ((work->w_state == WORK_CANCELLED) && (work->w_wq == wq));
+}
+
+static void
+linux_wait_for_invoked_work(struct work_struct *work)
+{
+ struct workqueue_struct *wq;
+
+ KASSERT(linux_work_locked(work));
+ KASSERT(work->w_state == WORK_INVOKED);
+
+ wq = work->w_wq;
+ mutex_enter(&wq->wq_lock);
+ linux_work_unlock(work);
+ while (wq->wq_current_work == work)
+ cv_wait(&wq->wq_cv, &wq->wq_lock);
+ mutex_exit(&wq->wq_lock);
+
+ linux_work_lock(work); /* XXX needless relock */
+}
+
+static void
+linux_worker(struct work *wk, void *arg)
+{
+ struct work_struct *const work = container_of(wk, struct work_struct,
+ w_wk);
+ struct workqueue_struct *const wq = arg;
+
+ linux_work_lock(work);
+ switch (work->w_state) {
+ case WORK_IDLE:
+ panic("idle work %p got queued: %p", work, wq);
+ break;
+
+ case WORK_DELAYED:
+ panic("delayed work %p got queued: %p", work, wq);
+ break;
+
+ case WORK_PENDING:
+ /* Get ready to invoke this one. */
+ mutex_enter(&wq->wq_lock);
+ work->w_state = WORK_INVOKED;
+ KASSERT(wq->wq_current_work == NULL);
+ wq->wq_current_work = work;
+ mutex_exit(&wq->wq_lock);
+
+ /* Unlock it and do it. Can't use work after this. */
+ linux_work_unlock(work);
+ (*work->w_fn)(work);
+
+ /* All done. Notify anyone waiting for completion. */
+ mutex_enter(&wq->wq_lock);
+ KASSERT(wq->wq_current_work == work);
+ wq->wq_current_work = NULL;
+ cv_broadcast(&wq->wq_cv);
+ mutex_exit(&wq->wq_lock);
+ return;
+
+ case WORK_INVOKED:
+ panic("invoked work %p got requeued: %p", work, wq);
+ break;
+
+ case WORK_CANCELLED:
+ /* Return to idle; notify anyone waiting for cancellation. */
+ mutex_enter(&wq->wq_lock);
+ work->w_state = WORK_IDLE;
+ work->w_wq = NULL;
+ cv_broadcast(&wq->wq_cv);
+ mutex_exit(&wq->wq_lock);
+ break;
+
+ case WORK_DELAYED_CANCELLED:
+ panic("cancelled delayed work %p got uqeued: %p", work, wq);
+ break;
+
+ default:
+ panic("work %p in bad state: %d", work, (int)work->w_state);
+ break;
+ }
+ linux_work_unlock(work);
+}
+
+/*
+ * Delayed work
+ */
+
+void
+INIT_DELAYED_WORK(struct delayed_work *dw, void (*fn)(struct work_struct *))
+{
+ INIT_WORK(&dw->work, fn);
+}
+
+void
+schedule_delayed_work(struct delayed_work *dw, unsigned long ticks)
+{
+ queue_delayed_work(system_wq, dw, ticks);
+}
+
+void
+queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *dw,
+ unsigned long ticks)
+{
+
+ KASSERT(wq != NULL);
+
+ /* If we're not actually delaying, queue the work now. */
+ if (ticks == 0) {
+ queue_work(wq, &dw->work);
+ return;
+ }
+
+ linux_work_lock(&dw->work);
+ switch (dw->work.w_state) {
+ case WORK_IDLE:
+ case WORK_INVOKED:
+ callout_init(&dw->dw_callout, CALLOUT_MPSAFE);
+ callout_reset(&dw->dw_callout, ticks, &linux_worker_intr, dw);
+ dw->work.w_state = WORK_DELAYED;
+ dw->work.w_wq = wq;
+ break;
+
+ case WORK_DELAYED:
+ callout_schedule(&dw->dw_callout, ticks);
+ break;
+
+ case WORK_PENDING:
+ KASSERT(dw->work.w_wq == wq);
+ break;
+
+ case WORK_CANCELLED:
+ case WORK_DELAYED_CANCELLED:
+ /* XXX Wait for cancellation and then queue? */
+ break;
+
+ default:
+ panic("delayed work %p in bad state: %d", dw,
+ (int)dw->work.w_state);
+ break;
+ }
+ linux_work_unlock(&dw->work);
+}
+
+bool
+cancel_delayed_work(struct delayed_work *dw)
+{
+ bool cancelled_p = false;
+
+ linux_work_lock(&dw->work);
+ switch (dw->work.w_state) {
+ case WORK_IDLE: /* Nothing to do. */
+ break;
+
+ case WORK_DELAYED:
+ dw->work.w_state = WORK_DELAYED_CANCELLED;
+ linux_cancel_delayed_work_callout(dw, false);
+ cancelled_p = true;
+ break;
+
+ case WORK_PENDING:
+ dw->work.w_state = WORK_CANCELLED;
+ cancelled_p = true;
+ break;
+
+ case WORK_INVOKED: /* Don't wait! */
+ break;
+
+ case WORK_CANCELLED: /* Already done. */
+ case WORK_DELAYED_CANCELLED:
+ break;
+
+ default:
+ panic("delayed work %p in bad state: %d", dw,
+ (int)dw->work.w_state);
+ break;
+ }
+ linux_work_unlock(&dw->work);
+
+ return cancelled_p;
+}
+
+bool
+cancel_delayed_work_sync(struct delayed_work *dw)
+{
+ bool cancelled_p = false;
+
+ linux_work_lock(&dw->work);
+ switch (dw->work.w_state) {
+ case WORK_IDLE: /* Nothing to do. */
+ break;
+
+ case WORK_DELAYED:
+ dw->work.w_state = WORK_DELAYED_CANCELLED;
+ linux_cancel_delayed_work_callout(dw, true);
+ cancelled_p = true;
+ break;
+
+ case WORK_PENDING:
+ dw->work.w_state = WORK_CANCELLED;
+ linux_wait_for_cancelled_work(&dw->work);
+ cancelled_p = true;
+
+ case WORK_INVOKED:
+ linux_wait_for_invoked_work(&dw->work);
+ break;
+
+ case WORK_CANCELLED: /* Already done. */
+ break;
+
+ case WORK_DELAYED_CANCELLED:
+ linux_wait_for_delayed_cancelled_work(dw);
+ break;
+
+ default:
+ panic("delayed work %p in bad state: %d", dw,
+ (int)dw->work.w_state);
+ break;
+ }
+ linux_work_unlock(&dw->work);
+
+ return cancelled_p;
+}
+
+static void
+linux_cancel_delayed_work_callout(struct delayed_work *dw, bool wait)
+{
+ bool fired_p;
+
+ KASSERT(linux_work_locked(&dw->work));
+ KASSERT(dw->work.w_state == WORK_DELAYED_CANCELLED);
+
+ if (wait) {
+ /*
+ * We unlock, halt, and then relock, rather than
+ * passing an interlock to callout_halt, for two
+ * reasons:
+ *
+ * (1) The work lock is not a mutex(9), so we can't use it.
+ * (2) The WORK_DELAYED_CANCELLED state serves as an interlock.
+ */
+ linux_work_unlock(&dw->work);
+ fired_p = callout_halt(&dw->dw_callout, NULL);
+ linux_work_lock(&dw->work);
+ } else {
+ fired_p = callout_stop(&dw->dw_callout);
+ }
+
+ /*
+ * fired_p means we didn't cancel the callout, so it must have
+ * already begun and will clean up after itself.
+ *
+ * !fired_p means we cancelled it so we have to clean up after
+ * it. Nobody else should have changed the state in that case.
+ */
+ if (!fired_p) {
+ struct workqueue_struct *wq;
+
+ KASSERT(linux_work_locked(&dw->work));
+ KASSERT(dw->work.w_state == WORK_DELAYED_CANCELLED);
+
+ wq = dw->work.w_wq;
+ mutex_enter(&wq->wq_lock);
+ TAILQ_REMOVE(&wq->wq_delayed, dw, dw_entry);
+ callout_destroy(&dw->dw_callout);
+ dw->work.w_state = WORK_IDLE;
+ dw->work.w_wq = NULL;
+ cv_broadcast(&wq->wq_cv);
+ mutex_exit(&wq->wq_lock);
+ }
+}
+
+static void
+linux_wait_for_delayed_cancelled_work(struct delayed_work *dw)
+{
+ struct workqueue_struct *wq;
+
+ KASSERT(linux_work_locked(&dw->work));
+ KASSERT(dw->work.w_state == WORK_DELAYED_CANCELLED);
+
+ wq = dw->work.w_wq;
+ do {
+ mutex_enter(&wq->wq_lock);
+ linux_work_unlock(&dw->work);
+ cv_wait(&wq->wq_cv, &wq->wq_lock);
+ mutex_exit(&wq->wq_lock);
+ linux_work_lock(&dw->work);
+ } while ((dw->work.w_state == WORK_DELAYED_CANCELLED) &&
+ (dw->work.w_wq == wq));
+}
+
+static void
+linux_worker_intr(void *arg)
+{
+ struct delayed_work *dw = arg;
+ struct workqueue_struct *wq;
+
+ linux_work_lock(&dw->work);
+
+ KASSERT((dw->work.w_state == WORK_DELAYED) ||
+ (dw->work.w_state == WORK_DELAYED_CANCELLED));
+
+ wq = dw->work.w_wq;
+ mutex_enter(&wq->wq_lock);
+
+ /* Queue the work, or return it to idle and alert any cancellers. */
+ if (__predict_true(dw->work.w_state == WORK_DELAYED)) {
+ dw->work.w_state = WORK_PENDING;
+ workqueue_enqueue(dw->work.w_wq->wq_workqueue, &dw->work.w_wk,
+ NULL);
+ } else {
+ KASSERT(dw->work.w_state == WORK_DELAYED_CANCELLED);
+ dw->work.w_state = WORK_IDLE;
+ dw->work.w_wq = NULL;
+ cv_broadcast(&wq->wq_cv);
+ }
+
+ /* Either way, the callout is done. */
+ TAILQ_REMOVE(&dw->work.w_wq->wq_delayed, dw, dw_entry);
+ callout_destroy(&dw->dw_callout);
+
+ mutex_exit(&wq->wq_lock);
+ linux_work_unlock(&dw->work);
+}
