Module Name: src
Committed By: rmind
Date: Sun Jun 2 02:20:05 UTC 2013
Modified Files:
src/sys/modules/npf: Makefile
src/sys/net/npf: files.npf npf.c npf_alg.c npf_alg_icmp.c npf_ctl.c
npf_impl.h npf_inet.c npf_nat.c npf_session.c
src/sys/rump/net/lib/libnpf: Makefile
Added Files:
src/sys/net/npf: npf_worker.c
Log Message:
- NPF connection tracking: rework synchronisation on tracking disable/enable
points and document it. Split the worker thread into a separate module
with an interface, so it could be re-used for other tasks.
- Replace ALG list with arrays and thus hit fewer cache lines.
- Misc bug fixes.
To generate a diff of this commit:
cvs rdiff -u -r1.12 -r1.13 src/sys/modules/npf/Makefile
cvs rdiff -u -r1.12 -r1.13 src/sys/net/npf/files.npf
cvs rdiff -u -r1.15 -r1.16 src/sys/net/npf/npf.c
cvs rdiff -u -r1.8 -r1.9 src/sys/net/npf/npf_alg.c
cvs rdiff -u -r1.16 -r1.17 src/sys/net/npf/npf_alg_icmp.c
cvs rdiff -u -r1.25 -r1.26 src/sys/net/npf/npf_ctl.c
cvs rdiff -u -r1.30 -r1.31 src/sys/net/npf/npf_impl.h
cvs rdiff -u -r1.21 -r1.22 src/sys/net/npf/npf_inet.c
cvs rdiff -u -r1.19 -r1.20 src/sys/net/npf/npf_nat.c
cvs rdiff -u -r1.23 -r1.24 src/sys/net/npf/npf_session.c
cvs rdiff -u -r0 -r1.1 src/sys/net/npf/npf_worker.c
cvs rdiff -u -r1.6 -r1.7 src/sys/rump/net/lib/libnpf/Makefile
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/modules/npf/Makefile
diff -u src/sys/modules/npf/Makefile:1.12 src/sys/modules/npf/Makefile:1.13
--- src/sys/modules/npf/Makefile:1.12 Sat Feb 9 03:35:33 2013
+++ src/sys/modules/npf/Makefile Sun Jun 2 02:20:05 2013
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.12 2013/02/09 03:35:33 rmind Exp $
+# $NetBSD: Makefile,v 1.13 2013/06/02 02:20:05 rmind Exp $
.include "../Makefile.inc"
@@ -10,7 +10,7 @@ SRCS= npf.c npf_alg.c npf_conf.c npf_ct
SRCS+= npf_inet.c npf_instr.c npf_mbuf.c npf_nat.c
SRCS+= npf_processor.c npf_ruleset.c npf_rproc.c npf_sendpkt.c
SRCS+= npf_session.c npf_state.c npf_state_tcp.c
-SRCS+= npf_tableset.c npf_tableset_ptree.c
+SRCS+= npf_tableset.c npf_tableset_ptree.c npf_worker.c
CPPFLAGS+= -DINET6
Index: src/sys/net/npf/files.npf
diff -u src/sys/net/npf/files.npf:1.12 src/sys/net/npf/files.npf:1.13
--- src/sys/net/npf/files.npf:1.12 Tue Mar 12 20:47:48 2013
+++ src/sys/net/npf/files.npf Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-# $NetBSD: files.npf,v 1.12 2013/03/12 20:47:48 christos Exp $
+# $NetBSD: files.npf,v 1.13 2013/06/02 02:20:04 rmind Exp $
#
# Public Domain.
#
@@ -28,6 +28,7 @@ file net/npf/npf_state_tcp.c npf
file net/npf/npf_nat.c npf
file net/npf/npf_alg.c npf
file net/npf/npf_sendpkt.c npf
+file net/npf/npf_worker.c npf
# Built-in extensions.
file net/npf/npf_ext_log.c npf
Index: src/sys/net/npf/npf.c
diff -u src/sys/net/npf/npf.c:1.15 src/sys/net/npf/npf.c:1.16
--- src/sys/net/npf/npf.c:1.15 Sat Feb 9 03:35:31 2013
+++ src/sys/net/npf/npf.c Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: npf.c,v 1.15 2013/02/09 03:35:31 rmind Exp $ */
+/* $NetBSD: npf.c,v 1.16 2013/06/02 02:20:04 rmind Exp $ */
/*-
* Copyright (c) 2009-2013 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.15 2013/02/09 03:35:31 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf.c,v 1.16 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -88,6 +88,7 @@ npf_init(void)
npf_stats_percpu = percpu_alloc(NPF_STATS_SIZE);
npf_sysctl = NULL;
+ npf_worker_sysinit();
npf_tableset_sysinit();
npf_session_sysinit();
npf_nat_sysinit();
@@ -129,6 +130,9 @@ npf_fini(void)
npf_session_sysfini();
npf_tableset_sysfini();
+ /* Note: worker is the last. */
+ npf_worker_sysfini();
+
if (npf_sysctl) {
sysctl_teardown(&npf_sysctl);
}
Index: src/sys/net/npf/npf_alg.c
diff -u src/sys/net/npf/npf_alg.c:1.8 src/sys/net/npf/npf_alg.c:1.9
--- src/sys/net/npf/npf_alg.c:1.8 Wed Mar 20 00:29:47 2013
+++ src/sys/net/npf/npf_alg.c Sun Jun 2 02:20:04 2013
@@ -1,7 +1,7 @@
-/* $NetBSD: npf_alg.c,v 1.8 2013/03/20 00:29:47 christos Exp $ */
+/* $NetBSD: npf_alg.c,v 1.9 2013/06/02 02:20:04 rmind Exp $ */
/*-
- * Copyright (c) 2010 The NetBSD Foundation, Inc.
+ * Copyright (c) 2010-2013 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This material is based upon work partially supported by The
@@ -30,11 +30,11 @@
*/
/*
- * NPF interface for application level gateways (ALGs).
+ * NPF interface for the Application Level Gateways (ALGs).
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_alg.c,v 1.8 2013/03/20 00:29:47 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_alg.c,v 1.9 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -47,67 +47,63 @@ __KERNEL_RCSID(0, "$NetBSD: npf_alg.c,v
#include "npf_impl.h"
-/* NAT ALG structure for registration. */
+/*
+ * NAT ALG description structure. For more compact use of cache,
+ * the functions are separated in their own arrays. The number of
+ * ALGs is expected to be very small.
+ */
+
struct npf_alg {
- LIST_ENTRY(npf_alg) na_entry;
- const char * na_name;
- npf_alg_t * na_bptr;
- npf_alg_func_t na_match_func;
- npf_alg_func_t na_tr_func;
- npf_alg_sfunc_t na_se_func;
+ const char * na_name;
+ u_int na_slot;
};
-static LIST_HEAD(, npf_alg) nat_alg_list __cacheline_aligned;
-static kmutex_t nat_alg_lock __cacheline_aligned;
-static pserialize_t nat_alg_psz __cacheline_aligned;
+#define NPF_MAX_ALGS 8
+
+/* List of ALGs and the count. */
+static pserialize_t alg_psz __cacheline_aligned;
+static npf_alg_t alg_list[NPF_MAX_ALGS] __read_mostly;
+static u_int alg_count __read_mostly;
+
+/* Session, matching and translation functions. */
+static npf_alg_sfunc_t alg_sfunc[NPF_MAX_ALGS] __read_mostly;
+static npf_alg_func_t alg_mfunc[NPF_MAX_ALGS] __read_mostly;
+static npf_alg_func_t alg_tfunc[NPF_MAX_ALGS] __read_mostly;
+
+static const char alg_prefix[] = "npf_alg_";
+#define NPF_EXT_PREFLEN (sizeof(alg_prefix) - 1)
void
npf_alg_sysinit(void)
{
-
- mutex_init(&nat_alg_lock, MUTEX_DEFAULT, IPL_NONE);
- nat_alg_psz = pserialize_create();
- LIST_INIT(&nat_alg_list);
+ alg_psz = pserialize_create();
+ memset(&alg_list, 0, sizeof(alg_list));
+ alg_count = 0;
+
+ memset(&alg_mfunc, 0, sizeof(alg_mfunc));
+ memset(&alg_tfunc, 0, sizeof(alg_tfunc));
+ memset(&alg_sfunc, 0, sizeof(alg_sfunc));
}
void
npf_alg_sysfini(void)
{
-
- KASSERT(LIST_EMPTY(&nat_alg_list));
- pserialize_destroy(nat_alg_psz);
- mutex_destroy(&nat_alg_lock);
+ pserialize_destroy(alg_psz);
}
-static const char npf_alg_prefix[] = "npf_alg_";
-#define NPF_EXT_PREFLEN (sizeof(npf_alg_prefix) - 1)
-
static npf_alg_t *
-npf_alg_lookup(const char *name, bool autoload)
+npf_alg_lookup(const char *name)
{
- npf_alg_t *alg;
- char modname[64 + NPF_EXT_PREFLEN];
- int error;
+ KASSERT(npf_config_locked_p());
- KASSERT(mutex_owned(&nat_alg_lock));
+ for (u_int i = 0; i < alg_count; i++) {
+ npf_alg_t *alg = &alg_list[i];
+ const char *aname = alg->na_name;
-again:
- LIST_FOREACH(alg, &nat_alg_list, na_entry)
- if (strcmp(alg->na_name, name) == 0)
- break;
-
- if (alg != NULL || !autoload)
- return alg;
-
- mutex_exit(&nat_alg_lock);
- autoload = false;
- snprintf(modname, sizeof(modname), "%s%s", npf_alg_prefix, name);
- error = module_autoload(modname, MODULE_CLASS_MISC);
- mutex_enter(&nat_alg_lock);
-
- if (error)
- return NULL;
- goto again;
+ if (aname && strcmp(aname, name) == 0)
+ return alg;
+ }
+ return NULL;
}
npf_alg_t *
@@ -115,9 +111,19 @@ npf_alg_construct(const char *name)
{
npf_alg_t *alg;
- mutex_enter(&nat_alg_lock);
- alg = npf_alg_lookup(name, true);
- mutex_exit(&nat_alg_lock);
+ npf_config_enter();
+ if ((alg = npf_alg_lookup(name)) == NULL) {
+ char modname[NPF_EXT_PREFLEN + 64];
+ snprintf(modname, sizeof(modname), "%s%s", alg_prefix, name);
+ npf_config_exit();
+
+ if (module_autoload(modname, MODULE_CLASS_MISC) != 0) {
+ return NULL;
+ }
+ npf_config_enter();
+ alg = npf_alg_lookup(name);
+ }
+ npf_config_exit();
return alg;
}
@@ -129,23 +135,37 @@ npf_alg_register(const char *name, npf_a
npf_alg_sfunc_t sfunc)
{
npf_alg_t *alg;
+ u_int i;
- alg = kmem_zalloc(sizeof(npf_alg_t), KM_SLEEP);
- alg->na_name = name;
- alg->na_bptr = alg;
- alg->na_match_func = mfunc;
- alg->na_tr_func = tfunc;
- alg->na_se_func = sfunc;
-
- mutex_enter(&nat_alg_lock);
- if (npf_alg_lookup(name, false) != NULL) {
- mutex_exit(&nat_alg_lock);
- kmem_free(alg, sizeof(npf_alg_t));
+ npf_config_enter();
+ if (npf_alg_lookup(name) != NULL) {
+ npf_config_exit();
+ return NULL;
+ }
+
+ /* Find a spare slot. */
+ for (i = 0; i < NPF_MAX_ALGS; i++) {
+ alg = &alg_list[i];
+ if (alg->na_name == NULL) {
+ break;
+ }
+ }
+ if (i == NPF_MAX_ALGS) {
+ npf_config_exit();
return NULL;
}
- LIST_INSERT_HEAD(&nat_alg_list, alg, na_entry);
- mutex_exit(&nat_alg_lock);
+ /* Register the ALG. */
+ alg->na_name = name;
+ alg->na_slot = i;
+
+ /* Assign the functions. */
+ alg_mfunc[i] = mfunc;
+ alg_tfunc[i] = tfunc;
+ alg_sfunc[i] = sfunc;
+
+ alg_count = MAX(alg_count, i + 1);
+ npf_config_exit();
return alg;
}
@@ -155,16 +175,20 @@ npf_alg_register(const char *name, npf_a
int
npf_alg_unregister(npf_alg_t *alg)
{
- mutex_enter(&nat_alg_lock);
- LIST_REMOVE(alg, na_entry);
- pserialize_perform(nat_alg_psz);
- mutex_exit(&nat_alg_lock);
+ u_int i = alg->na_slot;
+ /* Deactivate the functions first. */
npf_config_enter();
+ alg_mfunc[i] = NULL;
+ alg_tfunc[i] = NULL;
+ alg_sfunc[i] = NULL;
+ pserialize_perform(alg_psz);
+
+ /* Finally, unregister the ALG. */
npf_ruleset_freealg(npf_config_natset(), alg);
+ alg->na_name = NULL;
npf_config_exit();
- kmem_free(alg, sizeof(npf_alg_t));
return 0;
}
@@ -174,13 +198,12 @@ npf_alg_unregister(npf_alg_t *alg)
bool
npf_alg_match(npf_cache_t *npc, nbuf_t *nbuf, npf_nat_t *nt, int di)
{
- npf_alg_t *alg;
bool match = false;
int s;
s = pserialize_read_enter();
- LIST_FOREACH(alg, &nat_alg_list, na_entry) {
- npf_alg_func_t func = alg->na_match_func;
+ for (u_int i = 0; i < alg_count; i++) {
+ npf_alg_func_t func = alg_mfunc[i];
if (func && func(npc, nbuf, nt, di)) {
match = true;
@@ -197,15 +220,14 @@ npf_alg_match(npf_cache_t *npc, nbuf_t *
void
npf_alg_exec(npf_cache_t *npc, nbuf_t *nbuf, npf_nat_t *nt, int di)
{
- npf_alg_t *alg;
int s;
s = pserialize_read_enter();
- LIST_FOREACH(alg, &nat_alg_list, na_entry) {
+ for (u_int i = 0; i < alg_count; i++) {
npf_alg_func_t func;
- if ((func = alg->na_tr_func) != NULL) {
- (func)(npc, nbuf, nt, di);
+ if ((func = alg_tfunc[i]) != NULL) {
+ func(npc, nbuf, nt, di);
}
}
pserialize_read_exit(s);
@@ -215,12 +237,11 @@ npf_session_t *
npf_alg_session(npf_cache_t *npc, nbuf_t *nbuf, int di)
{
npf_session_t *se = NULL;
- npf_alg_t *alg;
int s;
s = pserialize_read_enter();
- LIST_FOREACH(alg, &nat_alg_list, na_entry) {
- npf_alg_sfunc_t func = alg->na_se_func;
+ for (u_int i = 0; i < alg_count; i++) {
+ npf_alg_sfunc_t func = alg_sfunc[i];
if (func && (se = func(npc, nbuf, di)) != NULL) {
break;
Index: src/sys/net/npf/npf_alg_icmp.c
diff -u src/sys/net/npf/npf_alg_icmp.c:1.16 src/sys/net/npf/npf_alg_icmp.c:1.17
--- src/sys/net/npf/npf_alg_icmp.c:1.16 Wed Mar 20 00:29:47 2013
+++ src/sys/net/npf/npf_alg_icmp.c Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: npf_alg_icmp.c,v 1.16 2013/03/20 00:29:47 christos Exp $ */
+/* $NetBSD: npf_alg_icmp.c,v 1.17 2013/06/02 02:20:04 rmind Exp $ */
/*-
* Copyright (c) 2010 The NetBSD Foundation, Inc.
@@ -34,11 +34,10 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_alg_icmp.c,v 1.16 2013/03/20 00:29:47 christos Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_alg_icmp.c,v 1.17 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/module.h>
-#include <sys/pool.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
@@ -78,17 +77,14 @@ static npf_session_t *npfa_icmp_session(
static int
npf_alg_icmp_init(void)
{
-
alg_icmp = npf_alg_register("icmp", npfa_icmp_match,
npfa_icmp_nat, npfa_icmp_session);
- KASSERT(alg_icmp != NULL);
- return 0;
+ return alg_icmp ? 0 : ENOMEM;
}
static int
npf_alg_icmp_fini(void)
{
-
KASSERT(alg_icmp != NULL);
return npf_alg_unregister(alg_icmp);
}
@@ -96,7 +92,6 @@ npf_alg_icmp_fini(void)
static int
npf_alg_icmp_modcmd(modcmd_t cmd, void *arg)
{
-
switch (cmd) {
case MODULE_CMD_INIT:
return npf_alg_icmp_init();
Index: src/sys/net/npf/npf_ctl.c
diff -u src/sys/net/npf/npf_ctl.c:1.25 src/sys/net/npf/npf_ctl.c:1.26
--- src/sys/net/npf/npf_ctl.c:1.25 Sun May 19 20:45:34 2013
+++ src/sys/net/npf/npf_ctl.c Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: npf_ctl.c,v 1.25 2013/05/19 20:45:34 rmind Exp $ */
+/* $NetBSD: npf_ctl.c,v 1.26 2013/06/02 02:20:04 rmind Exp $ */
/*-
* Copyright (c) 2009-2013 The NetBSD Foundation, Inc.
@@ -37,7 +37,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_ctl.c,v 1.25 2013/05/19 20:45:34 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_ctl.c,v 1.26 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/conf.h>
@@ -743,36 +743,36 @@ npfctl_sessions_load(u_long cmd, void *d
/* Create a session hash table. */
sehasht = sess_htable_create();
- if (sehasht == NULL) {
- prop_object_release(selist);
- return ENOMEM;
- }
/*
* Iterate through and construct each session. Note: acquire the
* config lock as we access NAT policies during the restore.
*/
error = 0;
- npf_config_enter();
it = prop_array_iterator(selist);
+
+ npf_config_enter();
while ((sedict = prop_object_iterator_next(it)) != NULL) {
/* Session - dictionary. */
if (prop_object_type(sedict) != PROP_TYPE_DICTIONARY) {
error = EINVAL;
- goto fail;
+ break;
}
/* Construct and insert real session structure. */
error = npf_session_restore(sehasht, sedict);
if (error) {
- goto fail;
+ break;
}
}
- sess_htable_reload(sehasht);
-fail:
npf_config_exit();
+
prop_object_iterator_release(it);
prop_object_release(selist);
- if (error) {
+
+ if (!error) {
+ /* Finally, load the new table. */
+ npf_session_load(sehasht);
+ } else {
/* Destroy session table. */
sess_htable_destroy(sehasht);
}
Index: src/sys/net/npf/npf_impl.h
diff -u src/sys/net/npf/npf_impl.h:1.30 src/sys/net/npf/npf_impl.h:1.31
--- src/sys/net/npf/npf_impl.h:1.30 Sun May 19 20:45:34 2013
+++ src/sys/net/npf/npf_impl.h Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: npf_impl.h,v 1.30 2013/05/19 20:45:34 rmind Exp $ */
+/* $NetBSD: npf_impl.h,v 1.31 2013/06/02 02:20:04 rmind Exp $ */
/*-
* Copyright (c) 2009-2013 The NetBSD Foundation, Inc.
@@ -96,6 +96,7 @@ typedef npf_table_t * npf_tableset_t;
typedef bool (*npf_alg_func_t)(npf_cache_t *, nbuf_t *, npf_nat_t *, int);
typedef npf_session_t *(*npf_alg_sfunc_t)(npf_cache_t *, nbuf_t *, int);
+typedef void (*npf_workfunc_t)(void);
#define NPF_NCODE_LIMIT 1024
#define NPF_TABLE_SLOTS 32
@@ -143,6 +144,12 @@ npf_tableset_t *npf_config_tableset(void
prop_dictionary_t npf_config_dict(void);
bool npf_default_pass(void);
+int npf_worker_sysinit(void);
+void npf_worker_sysfini(void);
+void npf_worker_signal(void);
+void npf_worker_register(npf_workfunc_t);
+void npf_worker_unregister(npf_workfunc_t);
+
void npflogattach(int);
void npflogdetach(void);
int npfctl_switch(void *);
@@ -272,11 +279,10 @@ void npf_rproc_run(npf_cache_t *, nbuf_
/* Session handling interface. */
void npf_session_sysinit(void);
void npf_session_sysfini(void);
-int npf_session_tracking(bool);
+void npf_session_tracking(bool);
npf_sehash_t * sess_htable_create(void);
void sess_htable_destroy(npf_sehash_t *);
-void sess_htable_reload(npf_sehash_t *);
npf_session_t * npf_session_lookup(const npf_cache_t *, const nbuf_t *,
const int, bool *);
@@ -289,6 +295,7 @@ void npf_session_setpass(npf_session_t
int npf_session_setnat(npf_session_t *, npf_nat_t *, const int);
npf_nat_t * npf_session_retnat(npf_session_t *, const int, bool *);
+void npf_session_load(npf_sehash_t *);
int npf_session_save(prop_array_t, prop_array_t);
int npf_session_restore(npf_sehash_t *, prop_dictionary_t);
Index: src/sys/net/npf/npf_inet.c
diff -u src/sys/net/npf/npf_inet.c:1.21 src/sys/net/npf/npf_inet.c:1.22
--- src/sys/net/npf/npf_inet.c:1.21 Sat Feb 9 03:35:32 2013
+++ src/sys/net/npf/npf_inet.c Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: npf_inet.c,v 1.21 2013/02/09 03:35:32 rmind Exp $ */
+/* $NetBSD: npf_inet.c,v 1.22 2013/06/02 02:20:04 rmind Exp $ */
/*-
* Copyright (c) 2009-2012 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
/*
- * Various procotol related helper routines.
+ * Various protocol related helper routines.
*
* This layer manipulates npf_cache_t structure i.e. caches requested headers
* and stores which information was cached in the information bit field.
@@ -39,7 +39,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_inet.c,v 1.21 2013/02/09 03:35:32 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_inet.c,v 1.22 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/types.h>
Index: src/sys/net/npf/npf_nat.c
diff -u src/sys/net/npf/npf_nat.c:1.19 src/sys/net/npf/npf_nat.c:1.20
--- src/sys/net/npf/npf_nat.c:1.19 Sat Feb 9 03:35:32 2013
+++ src/sys/net/npf/npf_nat.c Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-/* $NetBSD: npf_nat.c,v 1.19 2013/02/09 03:35:32 rmind Exp $ */
+/* $NetBSD: npf_nat.c,v 1.20 2013/06/02 02:20:04 rmind Exp $ */
/*-
* Copyright (c) 2010-2013 The NetBSD Foundation, Inc.
@@ -76,7 +76,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_nat.c,v 1.19 2013/02/09 03:35:32 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_nat.c,v 1.20 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -260,7 +260,8 @@ npf_nat_freepolicy(npf_natpolicy_t *np)
}
mutex_exit(&np->n_lock);
- /* All references should be going away. */
+ /* Kick the worker - all references should be going away. */
+ npf_worker_signal();
while (np->n_refcnt) {
kpause("npfgcnat", false, 1, NULL);
}
@@ -827,7 +828,10 @@ npf_nat_restore(prop_dictionary_t sedict
return NULL;
}
- /* Match if there is an existing NAT policy. */
+ /*
+ * Match if there is an existing NAT policy. Will acquire the
+ * reference on it if further operations are successful.
+ */
KASSERT(npf_config_locked_p());
rl = npf_ruleset_matchnat(npf_config_natset(), __UNCONST(onp));
if (rl == NULL) {
@@ -840,6 +844,7 @@ npf_nat_restore(prop_dictionary_t sedict
if (!npf_nat_takeport(np, ntraw->nt_tport)) {
return NULL;
}
+ atomic_inc_uint(&np->n_refcnt);
/* Create and return NAT entry for association. */
nt = pool_cache_get(nat_cache, PR_WAITOK);
Index: src/sys/net/npf/npf_session.c
diff -u src/sys/net/npf/npf_session.c:1.23 src/sys/net/npf/npf_session.c:1.24
--- src/sys/net/npf/npf_session.c:1.23 Mon Mar 18 00:17:20 2013
+++ src/sys/net/npf/npf_session.c Sun Jun 2 02:20:04 2013
@@ -1,7 +1,7 @@
-/* $NetBSD: npf_session.c,v 1.23 2013/03/18 00:17:20 rmind Exp $ */
+/* $NetBSD: npf_session.c,v 1.24 2013/06/02 02:20:04 rmind Exp $ */
/*-
- * Copyright (c) 2010-2012 The NetBSD Foundation, Inc.
+ * Copyright (c) 2010-2013 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This material is based upon work partially supported by The
@@ -65,22 +65,34 @@
* and should be released by the caller. Reference guarantees that the
* session will not be destroyed, although it may be expired.
*
- * Querying ALGs
+ * Synchronisation
*
- * Application-level gateways (ALGs) can inspect the packet and
- * determine whether the packet matches an ALG case. An ALG may
- * also lookup a session using different identifiers and return.
- * the packet cache (npf_cache_t) representing the IDs.
+ * Session hash table is accessed in a lock-less manner by the main
+ * operations: npf_session_inspect() and npf_session_establish().
+ * Since they are always called from a software interrupt, the hash
+ * table is protected using passive serialisation. The main place
+ * which can destroy the hash table is npf_session_reload(). It has
+ * to synchronise with other readers and writers using sess_lock,
+ * primarily the G/C thread.
+ *
+ * ALG support
+ *
+ * Application-level gateways (ALGs) can override generic session
+ * inspection (npf_alg_session() in npf_session_inspect() function)
+ * by performing their own lookup using different identifiers.
+ * Recursive call to npf_session_inspect() is not allowed, they
+ * ought to use npf_session_lookup() for this purpose.
*
* Lock order
*
- * [ sess_lock -> ]
- * npf_sehash_t::sh_lock ->
- * npf_state_t::nst_lock
+ * sess_lock ->
+ * [ npf_config_lock -> ]
+ * npf_sehash_t::sh_lock ->
+ * npf_state_t::nst_lock
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: npf_session.c,v 1.23 2013/03/18 00:17:20 rmind Exp $");
+__KERNEL_RCSID(0, "$NetBSD: npf_session.c,v 1.24 2013/06/02 02:20:04 rmind Exp $");
#include <sys/param.h>
#include <sys/types.h>
@@ -180,60 +192,53 @@ CTASSERT(PFIL_ALL == (0x001 | 0x002));
* Session tracking state: disabled (off), enabled (on) or flush request.
*/
enum { SESS_TRACKING_OFF, SESS_TRACKING_ON, SESS_TRACKING_FLUSH };
-static int sess_tracking __cacheline_aligned;
+static volatile int sess_tracking __cacheline_aligned;
-/* Session hash table, lock and session cache. */
+/* Session hash table, session cache and the lock. */
static npf_sehash_t * sess_hashtbl __read_mostly;
static pool_cache_t sess_cache __read_mostly;
+static kmutex_t sess_lock __cacheline_aligned;
+static kcondvar_t sess_cv __cacheline_aligned;
+static struct npf_sesslist sess_gc_list __cacheline_aligned;
-static kmutex_t sess_lock;
-static kcondvar_t sess_cv;
-static lwp_t * sess_gc_lwp;
-
-#define SESS_GC_INTERVAL 5 /* 5 sec */
-
-static void sess_tracking_stop(void);
+static void npf_session_worker(void);
static void npf_session_destroy(npf_session_t *);
-static void npf_session_worker(void *) __dead;
/*
* npf_session_sys{init,fini}: initialise/destroy session handling structures.
*
- * Session table and G/C thread are initialised when session tracking gets
- * actually enabled via npf_session_tracking() interface.
+ * Session table is initialised when session tracking gets enabled via
+ * npf_session_tracking() interface.
*/
void
npf_session_sysinit(void)
{
-
sess_cache = pool_cache_init(sizeof(npf_session_t), coherency_unit,
0, 0, "npfsespl", NULL, IPL_NET, NULL, NULL, NULL);
mutex_init(&sess_lock, MUTEX_DEFAULT, IPL_NONE);
- cv_init(&sess_cv, "npfgccv");
+ cv_init(&sess_cv, "npfsecv");
+ sess_tracking = SESS_TRACKING_OFF;
+ LIST_INIT(&sess_gc_list);
sess_hashtbl = NULL;
- sess_gc_lwp = NULL;
- sess_tracking = SESS_TRACKING_OFF;
+ npf_worker_register(npf_session_worker);
}
void
npf_session_sysfini(void)
{
-
/* Disable tracking, flush all sessions. */
npf_session_tracking(false);
- KASSERT(sess_tracking == SESS_TRACKING_OFF);
- KASSERT(sess_gc_lwp == NULL);
+ npf_worker_unregister(npf_session_worker);
- /* Sessions might have been restored while the tracking is off. */
- if (sess_hashtbl) {
- sess_htable_destroy(sess_hashtbl);
- }
+ KASSERT(sess_tracking == SESS_TRACKING_OFF);
+ KASSERT(LIST_EMPTY(&sess_gc_list));
+ KASSERT(sess_hashtbl == NULL);
pool_cache_destroy(sess_cache);
- cv_destroy(&sess_cv);
mutex_destroy(&sess_lock);
+ cv_destroy(&sess_cv);
}
/*
@@ -314,142 +319,96 @@ sess_hash_bucket(npf_sehash_t *stbl, con
npf_sehash_t *
sess_htable_create(void)
{
- npf_sehash_t *stbl, *sh;
- u_int i;
+ npf_sehash_t *tbl;
+
+ tbl = kmem_zalloc(SESS_HASH_BUCKETS * sizeof(npf_sehash_t), KM_SLEEP);
+ for (u_int i = 0; i < SESS_HASH_BUCKETS; i++) {
+ npf_sehash_t *sh = &tbl[i];
- stbl = kmem_zalloc(SESS_HASH_BUCKETS * sizeof(*sh), KM_SLEEP);
- if (stbl == NULL) {
- return NULL;
- }
- for (i = 0; i < SESS_HASH_BUCKETS; i++) {
- sh = &stbl[i];
LIST_INIT(&sh->sh_list);
rb_tree_init(&sh->sh_tree, &sess_rbtree_ops);
rw_init(&sh->sh_lock);
sh->sh_count = 0;
}
- return stbl;
+ return tbl;
}
void
-sess_htable_destroy(npf_sehash_t *stbl)
+sess_htable_destroy(npf_sehash_t *tbl)
{
- npf_sehash_t *sh;
- u_int i;
+ for (u_int i = 0; i < SESS_HASH_BUCKETS; i++) {
+ npf_sehash_t *sh = &tbl[i];
- for (i = 0; i < SESS_HASH_BUCKETS; i++) {
- sh = &stbl[i];
KASSERT(sh->sh_count == 0);
KASSERT(LIST_EMPTY(&sh->sh_list));
KASSERT(!rb_tree_iterate(&sh->sh_tree, NULL, RB_DIR_LEFT));
rw_destroy(&sh->sh_lock);
}
- kmem_free(stbl, SESS_HASH_BUCKETS * sizeof(*sh));
+ kmem_free(tbl, SESS_HASH_BUCKETS * sizeof(npf_sehash_t));
}
-void
-sess_htable_reload(npf_sehash_t *stbl)
+/*
+ * npf_session_reload: perform reload by flushing the current hash table
+ * of the sessions and replacing with the new one or just destroying.
+ *
+ * Key routine synchronising with all other readers and writers.
+ */
+static void
+npf_session_reload(npf_sehash_t *newtbl, int tracking)
{
- npf_sehash_t *oldstbl;
+ npf_sehash_t *oldtbl;
- /* Flush all existing entries. */
+ /* Must synchronise with G/C thread and session saving/restoring. */
mutex_enter(&sess_lock);
- if (sess_gc_lwp) {
- sess_tracking = SESS_TRACKING_FLUSH;
- cv_broadcast(&sess_cv);
- }
while (sess_tracking == SESS_TRACKING_FLUSH) {
cv_wait(&sess_cv, &sess_lock);
}
- /* Set a new session table. */
- oldstbl = sess_hashtbl;
- sess_hashtbl = stbl;
- mutex_exit(&sess_lock);
+ /*
+ * Set the flush status. It disables session inspection as well as
+ * creation. There may be some operations in-flight, drain them.
+ */
+ npf_config_enter();
+ sess_tracking = SESS_TRACKING_FLUSH;
+ npf_config_sync();
+ npf_config_exit();
- /* Destroy the old table. */
- if (oldstbl) {
- sess_htable_destroy(oldstbl);
+ /* Notify the worker to G/C all sessions. */
+ npf_worker_signal();
+ while (sess_tracking == SESS_TRACKING_FLUSH) {
+ cv_wait(&sess_cv, &sess_lock);
}
-}
-
-/*
- * Session tracking routines. Note: manages tracking structures.
- */
-
-static int
-sess_tracking_start(void)
-{
- npf_sehash_t *nstbl;
- nstbl = sess_htable_create();
- if (nstbl == NULL) {
- return ENOMEM;
- }
+ /* Install the new hash table, make it visible. */
+ oldtbl = atomic_swap_ptr(&sess_hashtbl, newtbl);
+ membar_sync();
+ sess_tracking = tracking;
- /* Note: should be visible before thread start. */
- mutex_enter(&sess_lock);
- if (sess_tracking != SESS_TRACKING_OFF) {
- mutex_exit(&sess_lock);
- sess_htable_destroy(nstbl);
- return EEXIST;
- }
- sess_hashtbl = nstbl;
- sess_tracking = SESS_TRACKING_ON;
+ /* Done. Destroy the old table, if any. */
mutex_exit(&sess_lock);
-
- if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL,
- npf_session_worker, NULL, &sess_gc_lwp, "npfgc")) {
- sess_tracking_stop();
- return ENOMEM;
+ if (oldtbl) {
+ sess_htable_destroy(oldtbl);
}
- return 0;
-}
-
-static void
-sess_tracking_stop(void)
-{
- npf_sehash_t *stbl;
-
- mutex_enter(&sess_lock);
- if (sess_tracking == SESS_TRACKING_OFF) {
- mutex_exit(&sess_lock);
- return;
- }
-
- /* Notify G/C thread to flush all sessions. */
- sess_tracking = SESS_TRACKING_OFF;
- cv_broadcast(&sess_cv);
-
- /* Wait for the exit. */
- while (sess_gc_lwp != NULL) {
- cv_wait(&sess_cv, &sess_lock);
- }
- stbl = sess_hashtbl;
- sess_hashtbl = NULL;
- mutex_exit(&sess_lock);
-
- sess_htable_destroy(stbl);
- pool_cache_invalidate(sess_cache);
}
/*
* npf_session_tracking: enable/disable session tracking.
*/
-int
+void
npf_session_tracking(bool track)
{
-
if (sess_tracking == SESS_TRACKING_OFF && track) {
/* Disabled -> Enable. */
- return sess_tracking_start();
+ npf_sehash_t *newtbl = sess_htable_create();
+ npf_session_reload(newtbl, SESS_TRACKING_ON);
+ return;
}
if (sess_tracking == SESS_TRACKING_ON && !track) {
/* Enabled -> Disable. */
- sess_tracking_stop();
- return 0;
+ npf_session_reload(NULL, SESS_TRACKING_OFF);
+ pool_cache_invalidate(sess_cache);
+ return;
}
- return 0;
}
static bool
@@ -459,7 +418,7 @@ npf_session_trackable_p(const npf_cache_
* Check if session tracking is on. Also, if layer 3 and 4 are not
* cached - protocol is not supported or packet is invalid.
*/
- if (sess_tracking == SESS_TRACKING_OFF) {
+ if (sess_tracking != SESS_TRACKING_ON) {
return false;
}
if (!npf_iscached(npc, NPC_IP46) || !npf_iscached(npc, NPC_LAYER4)) {
@@ -929,14 +888,14 @@ static void
npf_session_gc(struct npf_sesslist *gc_list, bool flushall)
{
struct timespec tsnow;
- npf_sentry_t *sen, *nsen;
- npf_session_t *se;
u_int i;
+ KASSERT(mutex_owned(&sess_lock));
getnanouptime(&tsnow);
/* Scan each session entry in the hash table. */
for (i = 0; i < SESS_HASH_BUCKETS; i++) {
+ npf_sentry_t *sen, *nsen;
npf_sehash_t *sh;
sh = &sess_hashtbl[i];
@@ -947,6 +906,8 @@ npf_session_gc(struct npf_sesslist *gc_l
/* For each (left -> right) ... */
sen = rb_tree_iterate(&sh->sh_tree, NULL, RB_DIR_LEFT);
while (sen != NULL) {
+ npf_session_t *se;
+
/* Get session, pre-iterate, skip if not expired. */
se = sen->se_backptr;
nsen = rb_tree_iterate(&sh->sh_tree, sen, RB_DIR_RIGHT);
@@ -981,15 +942,33 @@ npf_session_gc(struct npf_sesslist *gc_l
}
/*
- * npf_session_freelist: destroy all sessions, which have no references,
- * in the given G/C list. Return true, if the list is empty.
+ * npf_session_worker: G/C to run from a worker thread.
*/
static void
-npf_session_freelist(struct npf_sesslist *gc_list)
+npf_session_worker(void)
{
npf_session_t *se, *nse;
- se = LIST_FIRST(gc_list);
+ /*
+ * Garbage collect expired sessions.
+ */
+ mutex_enter(&sess_lock);
+ if (sess_hashtbl) {
+ bool flush = (sess_tracking != SESS_TRACKING_ON);
+ npf_session_gc(&sess_gc_list, flush);
+ }
+ if (sess_tracking == SESS_TRACKING_FLUSH) {
+ /* Flush was requested - indicate we are done. */
+ sess_tracking = SESS_TRACKING_OFF;
+ cv_broadcast(&sess_cv);
+ }
+ mutex_exit(&sess_lock);
+again:
+ /*
+ * Destroy all sessions in the G/C list.
+ * May need to wait for the references to drain.
+ */
+ se = LIST_FIRST(&sess_gc_list);
while (se != NULL) {
nse = LIST_NEXT(se, s_list);
if (se->s_refcnt == 0) {
@@ -999,48 +978,17 @@ npf_session_freelist(struct npf_sesslist
}
se = nse;
}
+ if (!LIST_EMPTY(&sess_gc_list)) {
+ kpause("npfcongc", false, 1, NULL);
+ goto again;
+ }
}
-/*
- * npf_session_worker: G/C worker thread.
- */
-static void
-npf_session_worker(void *arg)
+void
+npf_session_load(npf_sehash_t *newtbl)
{
- struct npf_sesslist gc_list;
- bool flushreq = false;
-
- LIST_INIT(&gc_list);
- do {
- /* Periodically wake up, unless get notified. */
- mutex_enter(&sess_lock);
- (void)cv_timedwait(&sess_cv, &sess_lock, SESS_GC_INTERVAL);
- flushreq = (sess_tracking != SESS_TRACKING_ON);
- npf_session_gc(&gc_list, flushreq);
- if (sess_tracking == SESS_TRACKING_FLUSH) {
- /* Flush was requested - on again, notify waiter. */
- sess_tracking = SESS_TRACKING_ON;
- cv_broadcast(&sess_cv);
- }
- mutex_exit(&sess_lock);
-
- npf_session_freelist(&gc_list);
-
- } while (sess_tracking != SESS_TRACKING_OFF);
-
- /* Wait for any referenced sessions to be released. */
- while (!LIST_EMPTY(&gc_list)) {
- kpause("npfgcfr", false, 1, NULL);
- npf_session_freelist(&gc_list);
- }
-
- /* Notify that we are done. */
- mutex_enter(&sess_lock);
- sess_gc_lwp = NULL;
- cv_broadcast(&sess_cv);
- mutex_exit(&sess_lock);
-
- kthread_exit(0);
+ KASSERT(newtbl != NULL);
+ npf_session_reload(newtbl, SESS_TRACKING_ON);
}
/*
@@ -1054,18 +1002,16 @@ npf_session_save(prop_array_t selist, pr
npf_session_t *se;
int error = 0, i;
- /* If not tracking - empty. */
+ /*
+ * If not tracking - empty. Note: must acquire sess_lock to
+ * prevent from hash table destruction as well as expiring or
+ * removing of sessions by the G/C thread.
+ */
mutex_enter(&sess_lock);
- if (sess_tracking == SESS_TRACKING_OFF) {
+ if (sess_tracking != SESS_TRACKING_ON) {
mutex_exit(&sess_lock);
return 0;
}
-
- /*
- * Note: hold the session lock to prevent G/C thread from session
- * expiring and removing. Therefore, no need to exclusively lock
- * the entire hash table.
- */
for (i = 0; i < SESS_HASH_BUCKETS; i++) {
sh = &sess_hashtbl[i];
if (sh->sh_count == 0) {
Index: src/sys/rump/net/lib/libnpf/Makefile
diff -u src/sys/rump/net/lib/libnpf/Makefile:1.6 src/sys/rump/net/lib/libnpf/Makefile:1.7
--- src/sys/rump/net/lib/libnpf/Makefile:1.6 Tue Mar 12 21:12:47 2013
+++ src/sys/rump/net/lib/libnpf/Makefile Sun Jun 2 02:20:04 2013
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.6 2013/03/12 21:12:47 christos Exp $
+# $NetBSD: Makefile,v 1.7 2013/06/02 02:20:04 rmind Exp $
#
# Public Domain.
#
@@ -11,7 +11,7 @@ SRCS= npf.c npf_alg.c npf_conf.c npf_ctl
SRCS+= npf_inet.c npf_instr.c npf_mbuf.c npf_nat.c
SRCS+= npf_processor.c npf_ruleset.c npf_rproc.c npf_sendpkt.c
SRCS+= npf_session.c npf_state.c npf_state_tcp.c
-SRCS+= npf_tableset.c npf_tableset_ptree.c
+SRCS+= npf_tableset.c npf_tableset_ptree.c npf_worker.c
SRCS+= if_npflog.c
SRCS+= npf_alg_icmp.c
Added files:
Index: src/sys/net/npf/npf_worker.c
diff -u /dev/null src/sys/net/npf/npf_worker.c:1.1
--- /dev/null Sun Jun 2 02:20:05 2013
+++ src/sys/net/npf/npf_worker.c Sun Jun 2 02:20:04 2013
@@ -0,0 +1,161 @@
+/* $NetBSD: npf_worker.c,v 1.1 2013/06/02 02:20:04 rmind Exp $ */
+
+/*-
+ * Copyright (c) 2010-2013 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This material is based upon work partially supported by The
+ * NetBSD Foundation under a contract with Mindaugas Rasiukevicius.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__KERNEL_RCSID(0, "$NetBSD: npf_worker.c,v 1.1 2013/06/02 02:20:04 rmind Exp $");
+
+#include <sys/param.h>
+#include <sys/types.h>
+
+#include <sys/mutex.h>
+#include <sys/kernel.h>
+#include <sys/kthread.h>
+
+#include "npf_impl.h"
+
+#define NPF_MAX_WORKS 4
+#define WORKER_INTERVAL mstohz(5 * 1000)
+
+static kmutex_t worker_lock;
+static kcondvar_t worker_cv;
+static kcondvar_t worker_event_cv;
+static lwp_t * worker_lwp;
+static uint64_t worker_loop;
+
+static npf_workfunc_t work_funcs[NPF_MAX_WORKS];
+
+static void npf_worker(void *) __dead;
+
+int
+npf_worker_sysinit(void)
+{
+ mutex_init(&worker_lock, MUTEX_DEFAULT, IPL_SOFTNET);
+ cv_init(&worker_cv, "npfgccv");
+ cv_init(&worker_event_cv, "npfevcv");
+ worker_lwp = (lwp_t *)0xdeadbabe;
+ worker_loop = 1;
+
+ if (kthread_create(PRI_NONE, KTHREAD_MPSAFE | KTHREAD_MUSTJOIN, NULL,
+ npf_worker, NULL, &worker_lwp, "npfgc")) {
+ return ENOMEM;
+ }
+ return 0;
+}
+
+void
+npf_worker_sysfini(void)
+{
+ lwp_t *l = worker_lwp;
+
+ /* Notify the worker and wait for the exit. */
+ mutex_enter(&worker_lock);
+ worker_lwp = NULL;
+ cv_broadcast(&worker_cv);
+ mutex_exit(&worker_lock);
+ kthread_join(l);
+
+ /* LWP has exited, destroy the structures. */
+ cv_destroy(&worker_cv);
+ cv_destroy(&worker_event_cv);
+ mutex_destroy(&worker_lock);
+}
+
+void
+npf_worker_signal(void)
+{
+ mutex_enter(&worker_lock);
+ cv_signal(&worker_cv);
+ mutex_exit(&worker_lock);
+}
+
+static bool
+npf_worker_testset(npf_workfunc_t find, npf_workfunc_t set)
+{
+ for (u_int i = 0; i < NPF_MAX_WORKS; i++) {
+ if (work_funcs[i] == find) {
+ work_funcs[i] = set;
+ return true;
+ }
+ }
+ return false;
+}
+
+void
+npf_worker_register(npf_workfunc_t func)
+{
+ mutex_enter(&worker_lock);
+ npf_worker_testset(NULL, func);
+ mutex_exit(&worker_lock);
+}
+
+void
+npf_worker_unregister(npf_workfunc_t func)
+{
+ uint64_t l = worker_loop;
+
+ mutex_enter(&worker_lock);
+ npf_worker_testset(func, NULL);
+ while (worker_loop == l) {
+ cv_signal(&worker_cv);
+ cv_wait(&worker_event_cv, &worker_lock);
+ }
+ mutex_exit(&worker_lock);
+}
+
+static void
+npf_worker(void *arg)
+{
+ for (;;) {
+ const bool finish = (worker_lwp == NULL);
+ u_int i = NPF_MAX_WORKS;
+ npf_workfunc_t work;
+
+ /* Run the jobs. */
+ while (i--) {
+ if ((work = work_funcs[i]) != NULL) {
+ work();
+ }
+ }
+
+ /* Exit if requested and all jobs are done. */
+ if (finish) {
+ break;
+ }
+
+ /* Sleep and periodically wake up, unless we get notified. */
+ mutex_enter(&worker_lock);
+ worker_loop++;
+ cv_broadcast(&worker_event_cv);
+ cv_timedwait(&worker_cv, &worker_lock, WORKER_INTERVAL);
+ mutex_exit(&worker_lock);
+ }
+ kthread_exit(0);
+}
