Author: avg
Date: Tue Nov 26 09:26:18 2013
New Revision: 258630
URL: http://svnweb.freebsd.org/changeset/base/258630
Log:
734 taskq_dispatch_prealloc() desired
943 zio_interrupt ends up calling taskq_dispatch with TQ_SLEEP
illumos/illumos-gate@5aeb94743e3be0c51e86f73096334611ae3a058e
Essentially FreeBSD taskqueues already operate in a mode that
was added to Illumos with taskq_dispatch_ent change.
We even exposed the superior FreeBSD interface as taskq_dispatch_safe.
Now we just rename taskq_dispatch_safe to taskq_dispatch_ent and
struct struct ostask to taskq_ent_t, so that code differences will be
minimal.
After this change sys/cddl/compat/opensolaris/sys/taskq.h header is no
longer needed.
Note that this commit is not an MFV because the upstream change was not
individually committed to the vendor area.
MFC after: 8 days
Deleted:
head/sys/cddl/compat/opensolaris/sys/taskq.h
Modified:
head/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h
head/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c
head/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
head/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h
Modified: head/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h
==============================================================================
--- head/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h Tue Nov
26 09:14:39 2013 (r258629)
+++ head/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h Tue Nov
26 09:26:18 2013 (r258630)
@@ -23,6 +23,9 @@
* Copyright (c) 2013 by Delphix. All rights reserved.
* Copyright (c) 2012, Joyent, Inc. All rights reserved.
*/
+/*
+ * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
+ */
#ifndef _SYS_ZFS_CONTEXT_H
#define _SYS_ZFS_CONTEXT_H
@@ -367,6 +370,16 @@ typedef struct taskq taskq_t;
typedef uintptr_t taskqid_t;
typedef void (task_func_t)(void *);
+typedef struct taskq_ent {
+ struct taskq_ent *tqent_next;
+ struct taskq_ent *tqent_prev;
+ task_func_t *tqent_func;
+ void *tqent_arg;
+ uintptr_t tqent_flags;
+} taskq_ent_t;
+
+#define TQENT_FLAG_PREALLOC 0x1 /* taskq_dispatch_ent used */
+
#define TASKQ_PREPOPULATE 0x0001
#define TASKQ_CPR_SAFE 0x0002 /* Use CPR safe protocol */
#define TASKQ_DYNAMIC 0x0004 /* Use dynamic thread
scheduling */
@@ -378,6 +391,7 @@ typedef void (task_func_t)(void *);
#define TQ_NOQUEUE 0x02 /* Do not enqueue if can't
dispatch */
#define TQ_FRONT 0x08 /* Queue in front */
+
extern taskq_t *system_taskq;
extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t);
@@ -386,6 +400,8 @@ extern taskq_t *taskq_create(const char
#define taskq_create_sysdc(a, b, d, e, p, dc, f) \
(taskq_create(a, b, maxclsyspri, d, e, f))
extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
+extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t,
+ taskq_ent_t *);
extern void taskq_destroy(taskq_t *);
extern void taskq_wait(taskq_t *);
extern int taskq_member(taskq_t *, void *);
Modified: head/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c
==============================================================================
--- head/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c Tue Nov 26
09:14:39 2013 (r258629)
+++ head/cddl/contrib/opensolaris/lib/libzpool/common/taskq.c Tue Nov 26
09:26:18 2013 (r258630)
@@ -22,19 +22,15 @@
* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
+/*
+ * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
+ */
#include <sys/zfs_context.h>
int taskq_now;
taskq_t *system_taskq;
-typedef struct task {
- struct task *task_next;
- struct task *task_prev;
- task_func_t *task_func;
- void *task_arg;
-} task_t;
-
#define TASKQ_ACTIVE 0x00010000
struct taskq {
@@ -51,18 +47,18 @@ struct taskq {
int tq_maxalloc;
kcondvar_t tq_maxalloc_cv;
int tq_maxalloc_wait;
- task_t *tq_freelist;
- task_t tq_task;
+ taskq_ent_t *tq_freelist;
+ taskq_ent_t tq_task;
};
-static task_t *
+static taskq_ent_t *
task_alloc(taskq_t *tq, int tqflags)
{
- task_t *t;
+ taskq_ent_t *t;
int rv;
again: if ((t = tq->tq_freelist) != NULL && tq->tq_nalloc >= tq->tq_minalloc) {
- tq->tq_freelist = t->task_next;
+ tq->tq_freelist = t->tqent_next;
} else {
if (tq->tq_nalloc >= tq->tq_maxalloc) {
if (!(tqflags & KM_SLEEP))
@@ -87,7 +83,7 @@ again: if ((t = tq->tq_freelist) != NULL
}
mutex_exit(&tq->tq_lock);
- t = kmem_alloc(sizeof (task_t), tqflags & KM_SLEEP);
+ t = kmem_alloc(sizeof (taskq_ent_t), tqflags & KM_SLEEP);
mutex_enter(&tq->tq_lock);
if (t != NULL)
@@ -97,15 +93,15 @@ again: if ((t = tq->tq_freelist) != NULL
}
static void
-task_free(taskq_t *tq, task_t *t)
+task_free(taskq_t *tq, taskq_ent_t *t)
{
if (tq->tq_nalloc <= tq->tq_minalloc) {
- t->task_next = tq->tq_freelist;
+ t->tqent_next = tq->tq_freelist;
tq->tq_freelist = t;
} else {
tq->tq_nalloc--;
mutex_exit(&tq->tq_lock);
- kmem_free(t, sizeof (task_t));
+ kmem_free(t, sizeof (taskq_ent_t));
mutex_enter(&tq->tq_lock);
}
@@ -116,7 +112,7 @@ task_free(taskq_t *tq, task_t *t)
taskqid_t
taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t tqflags)
{
- task_t *t;
+ taskq_ent_t *t;
if (taskq_now) {
func(arg);
@@ -130,26 +126,58 @@ taskq_dispatch(taskq_t *tq, task_func_t
return (0);
}
if (tqflags & TQ_FRONT) {
- t->task_next = tq->tq_task.task_next;
- t->task_prev = &tq->tq_task;
+ t->tqent_next = tq->tq_task.tqent_next;
+ t->tqent_prev = &tq->tq_task;
} else {
- t->task_next = &tq->tq_task;
- t->task_prev = tq->tq_task.task_prev;
+ t->tqent_next = &tq->tq_task;
+ t->tqent_prev = tq->tq_task.tqent_prev;
}
- t->task_next->task_prev = t;
- t->task_prev->task_next = t;
- t->task_func = func;
- t->task_arg = arg;
+ t->tqent_next->tqent_prev = t;
+ t->tqent_prev->tqent_next = t;
+ t->tqent_func = func;
+ t->tqent_arg = arg;
cv_signal(&tq->tq_dispatch_cv);
mutex_exit(&tq->tq_lock);
return (1);
}
void
+taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags,
+ taskq_ent_t *t)
+{
+ ASSERT(func != NULL);
+ ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC));
+
+ /*
+ * Mark it as a prealloc'd task. This is important
+ * to ensure that we don't free it later.
+ */
+ t->tqent_flags |= TQENT_FLAG_PREALLOC;
+ /*
+ * Enqueue the task to the underlying queue.
+ */
+ mutex_enter(&tq->tq_lock);
+
+ if (flags & TQ_FRONT) {
+ t->tqent_next = tq->tq_task.tqent_next;
+ t->tqent_prev = &tq->tq_task;
+ } else {
+ t->tqent_next = &tq->tq_task;
+ t->tqent_prev = tq->tq_task.tqent_prev;
+ }
+ t->tqent_next->tqent_prev = t;
+ t->tqent_prev->tqent_next = t;
+ t->tqent_func = func;
+ t->tqent_arg = arg;
+ cv_signal(&tq->tq_dispatch_cv);
+ mutex_exit(&tq->tq_lock);
+}
+
+void
taskq_wait(taskq_t *tq)
{
mutex_enter(&tq->tq_lock);
- while (tq->tq_task.task_next != &tq->tq_task || tq->tq_active != 0)
+ while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0)
cv_wait(&tq->tq_wait_cv, &tq->tq_lock);
mutex_exit(&tq->tq_lock);
}
@@ -158,27 +186,32 @@ static void *
taskq_thread(void *arg)
{
taskq_t *tq = arg;
- task_t *t;
+ taskq_ent_t *t;
+ boolean_t prealloc;
mutex_enter(&tq->tq_lock);
while (tq->tq_flags & TASKQ_ACTIVE) {
- if ((t = tq->tq_task.task_next) == &tq->tq_task) {
+ if ((t = tq->tq_task.tqent_next) == &tq->tq_task) {
if (--tq->tq_active == 0)
cv_broadcast(&tq->tq_wait_cv);
cv_wait(&tq->tq_dispatch_cv, &tq->tq_lock);
tq->tq_active++;
continue;
}
- t->task_prev->task_next = t->task_next;
- t->task_next->task_prev = t->task_prev;
+ t->tqent_prev->tqent_next = t->tqent_next;
+ t->tqent_next->tqent_prev = t->tqent_prev;
+ t->tqent_next = NULL;
+ t->tqent_prev = NULL;
+ prealloc = t->tqent_flags & TQENT_FLAG_PREALLOC;
mutex_exit(&tq->tq_lock);
rw_enter(&tq->tq_threadlock, RW_READER);
- t->task_func(t->task_arg);
+ t->tqent_func(t->tqent_arg);
rw_exit(&tq->tq_threadlock);
mutex_enter(&tq->tq_lock);
- task_free(tq, t);
+ if (!prealloc)
+ task_free(tq, t);
}
tq->tq_nthreads--;
cv_broadcast(&tq->tq_wait_cv);
@@ -217,8 +250,8 @@ taskq_create(const char *name, int nthre
tq->tq_nthreads = nthreads;
tq->tq_minalloc = minalloc;
tq->tq_maxalloc = maxalloc;
- tq->tq_task.task_next = &tq->tq_task;
- tq->tq_task.task_prev = &tq->tq_task;
+ tq->tq_task.tqent_next = &tq->tq_task;
+ tq->tq_task.tqent_prev = &tq->tq_task;
tq->tq_threadlist = kmem_alloc(nthreads * sizeof (thread_t), KM_SLEEP);
if (flags & TASKQ_PREPOPULATE) {
Modified: head/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
==============================================================================
--- head/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Tue Nov 26
09:14:39 2013 (r258629)
+++ head/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Tue Nov 26
09:26:18 2013 (r258630)
@@ -46,7 +46,7 @@ static void
system_taskq_init(void *arg)
{
- taskq_zone = uma_zcreate("taskq_zone", sizeof(struct ostask),
+ taskq_zone = uma_zcreate("taskq_zone", sizeof(taskq_ent_t),
NULL, NULL, NULL, NULL, 0, 0);
system_taskq = taskq_create("system_taskq", mp_ncpus, 0, 0, 0, 0);
}
@@ -104,9 +104,9 @@ taskq_member(taskq_t *tq, kthread_t *thr
static void
taskq_run(void *arg, int pending __unused)
{
- struct ostask *task = arg;
+ taskq_ent_t *task = arg;
- task->ost_func(task->ost_arg);
+ task->tqent_func(task->tqent_arg);
uma_zfree(taskq_zone, task);
}
@@ -114,7 +114,7 @@ taskq_run(void *arg, int pending __unuse
taskqid_t
taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
{
- struct ostask *task;
+ taskq_ent_t *task;
int mflag, prio;
if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP)
@@ -131,26 +131,26 @@ taskq_dispatch(taskq_t *tq, task_func_t
if (task == NULL)
return (0);
- task->ost_func = func;
- task->ost_arg = arg;
+ task->tqent_func = func;
+ task->tqent_arg = arg;
- TASK_INIT(&task->ost_task, prio, taskq_run, task);
- taskqueue_enqueue(tq->tq_queue, &task->ost_task);
+ TASK_INIT(&task->tqent_task, prio, taskq_run, task);
+ taskqueue_enqueue(tq->tq_queue, &task->tqent_task);
return ((taskqid_t)(void *)task);
}
static void
-taskq_run_safe(void *arg, int pending __unused)
+taskq_run_ent(void *arg, int pending __unused)
{
- struct ostask *task = arg;
+ taskq_ent_t *task = arg;
- task->ost_func(task->ost_arg);
+ task->tqent_func(task->tqent_arg);
}
-taskqid_t
-taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg, u_int flags,
- struct ostask *task)
+void
+taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, u_int flags,
+ taskq_ent_t *task)
{
int prio;
@@ -160,11 +160,9 @@ taskq_dispatch_safe(taskq_t *tq, task_fu
*/
prio = !!(flags & TQ_FRONT);
- task->ost_func = func;
- task->ost_arg = arg;
+ task->tqent_func = func;
+ task->tqent_arg = arg;
- TASK_INIT(&task->ost_task, prio, taskq_run_safe, task);
- taskqueue_enqueue(tq->tq_queue, &task->ost_task);
-
- return ((taskqid_t)(void *)task);
+ TASK_INIT(&task->tqent_task, prio, taskq_run_ent, task);
+ taskqueue_enqueue(tq->tq_queue, &task->tqent_task);
}
Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c Tue Nov 26
09:14:39 2013 (r258629)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c Tue Nov 26
09:26:18 2013 (r258630)
@@ -825,7 +825,7 @@ static taskq_t *
spa_taskq_create(spa_t *spa, const char *name, enum zti_modes mode,
uint_t value)
{
- uint_t flags = TASKQ_PREPOPULATE;
+ uint_t flags = 0;
boolean_t batch = B_FALSE;
switch (mode) {
Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h Tue Nov
26 09:14:39 2013 (r258629)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h Tue Nov
26 09:26:18 2013 (r258630)
@@ -24,6 +24,7 @@
* Copyright (c) 2012 by Delphix. All rights reserved.
* Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
* Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
*/
#ifndef _ZIO_H
@@ -474,10 +475,9 @@ struct zio {
zio_cksum_report_t *io_cksum_report;
uint64_t io_ena;
-#ifdef _KERNEL
- /* FreeBSD only. */
- struct ostask io_task;
-#endif
+ /* Taskq dispatching state */
+ taskq_ent_t io_tqent;
+
avl_node_t io_trim_node;
list_node_t io_trim_link;
};
Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c Tue Nov 26
09:14:39 2013 (r258629)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c Tue Nov 26
09:26:18 2013 (r258630)
@@ -21,6 +21,7 @@
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
*/
#include <sys/zfs_context.h>
@@ -1224,7 +1225,7 @@ zio_taskq_dispatch(zio_t *zio, enum zio_
{
spa_t *spa = zio->io_spa;
zio_type_t t = zio->io_type;
- int flags = TQ_SLEEP | (cutinline ? TQ_FRONT : 0);
+ int flags = (cutinline ? TQ_FRONT : 0);
ASSERT(q == ZIO_TASKQ_ISSUE || q == ZIO_TASKQ_INTERRUPT);
@@ -1250,13 +1251,19 @@ zio_taskq_dispatch(zio_t *zio, enum zio_
q++;
ASSERT3U(q, <, ZIO_TASKQ_TYPES);
-#ifdef _KERNEL
- (void) taskq_dispatch_safe(spa->spa_zio_taskq[t][q],
- (task_func_t *)zio_execute, zio, flags, &zio->io_task);
+
+ /*
+ * NB: We are assuming that the zio can only be dispatched
+ * to a single taskq at a time. It would be a grievous error
+ * to dispatch the zio to another taskq at the same time.
+ */
+#if defined(illumos) || !defined(_KERNEL)
+ ASSERT(zio->io_tqent.tqent_next == NULL);
#else
- (void) taskq_dispatch(spa->spa_zio_taskq[t][q],
- (task_func_t *)zio_execute, zio, flags);
+ ASSERT(zio->io_tqent.tqent_task.ta_pending == 0);
#endif
+ taskq_dispatch_ent(spa->spa_zio_taskq[t][q],
+ (task_func_t *)zio_execute, zio, flags, &zio->io_tqent);
}
static boolean_t
@@ -3174,16 +3181,15 @@ zio_done(zio_t *zio)
* Reexecution is potentially a huge amount of work.
* Hand it off to the otherwise-unused claim taskq.
*/
-#ifdef _KERNEL
- (void) taskq_dispatch_safe(
- spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
- (task_func_t *)zio_reexecute, zio, TQ_SLEEP,
- &zio->io_task);
+#if defined(illumos) || !defined(_KERNEL)
+ ASSERT(zio->io_tqent.tqent_next == NULL);
#else
- (void) taskq_dispatch(
- spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
- (task_func_t *)zio_reexecute, zio, TQ_SLEEP);
+ ASSERT(zio->io_tqent.tqent_task.ta_pending == 0);
#endif
+ (void) taskq_dispatch_ent(
+ spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
+ (task_func_t *)zio_reexecute, zio, 0,
+ &zio->io_tqent);
}
return (ZIO_PIPELINE_STOP);
}
Modified: head/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Tue Nov 26
09:14:39 2013 (r258629)
+++ head/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Tue Nov 26
09:26:18 2013 (r258630)
@@ -45,6 +45,12 @@ typedef struct taskq taskq_t;
typedef uintptr_t taskqid_t;
typedef void (task_func_t)(void *);
+typedef struct taskq_ent {
+ struct task tqent_task;
+ task_func_t *tqent_func;
+ void *tqent_arg;
+} taskq_ent_t;
+
struct proc;
/*
@@ -80,6 +86,8 @@ taskq_t *taskq_create_proc(const char *,
taskq_t *taskq_create_sysdc(const char *, int, int, int,
struct proc *, uint_t, uint_t);
taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
+void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t,
+ taskq_ent_t *);
void nulltask(void *);
void taskq_destroy(taskq_t *);
void taskq_wait(taskq_t *);
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
