Module Name: src
Committed By: ozaki-r
Date: Tue Aug 8 04:17:34 UTC 2017
Modified Files:
src/sys/netipsec: key.c keydb.h
Log Message:
MP-ify SAD (key_sad.sahlist and sah entries)
localcount(9) is used to protect key_sad.sahlist and sah entries
as well as SPD (and will be used for SAD sav).
Please read the locking notes of SAD for more details.
To generate a diff of this commit:
cvs rdiff -u -r1.215 -r1.216 src/sys/netipsec/key.c
cvs rdiff -u -r1.18 -r1.19 src/sys/netipsec/keydb.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/netipsec/key.c
diff -u src/sys/netipsec/key.c:1.215 src/sys/netipsec/key.c:1.216
--- src/sys/netipsec/key.c:1.215 Tue Aug 8 01:56:49 2017
+++ src/sys/netipsec/key.c Tue Aug 8 04:17:34 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: key.c,v 1.215 2017/08/08 01:56:49 ozaki-r Exp $ */
+/* $NetBSD: key.c,v 1.216 2017/08/08 04:17:34 ozaki-r Exp $ */
/* $FreeBSD: src/sys/netipsec/key.c,v 1.3.2.3 2004/02/14 22:23:23 bms Exp $ */
/* $KAME: key.c,v 1.191 2001/06/27 10:46:49 sakane Exp $ */
@@ -32,7 +32,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.215 2017/08/08 01:56:49 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: key.c,v 1.216 2017/08/08 04:17:34 ozaki-r Exp $");
/*
* This code is referd to RFC 2367
@@ -198,12 +198,35 @@ static u_int32_t acq_seq = 0;
* until GC by the timer
*/
/*
+ * Locking notes on SAD:
+ * - Data structures
+ * - SAs are managed by the list called key_sad.sahlist and sav lists of sah
+ * entries
+ * - A sah has sav lists for each SA state
+ * - Multiple sahs with the same saidx can exist
+ * - Only one entry has MATURE state and others should be DEAD
+ * - DEAD entries are just ignored from searching
+ * - Modifications to the key_sad.sahlist must be done with holding key_sad.lock
+ * which is a adaptive mutex
+ * - Read accesses to the key_sad.sahlist must be in pserialize(9) read sections
+ * - sah's lifetime is managed by localcount(9)
+ * - Getting an sah entry
+ * - We get an SP from the key_spd.splist
+ * - Must iterate the list and increment the reference count of a found sah
+ * (by key_sah_ref) in a pserialize read section
+ * - A gotten sah must be released after use by key_sah_unref
+ * - An sah is destroyed when its state become DEAD and no sav is
+ * listed to the sah
+ * - The destruction is done only in the timer (see key_timehandler_sad)
+ */
+/*
* Locking notes on misc data:
* - All lists of key_misc are protected by key_misc.lock
* - key_misc.lock must be held even for read accesses
*/
-static pserialize_t key_psz __read_mostly;
+static pserialize_t key_spd_psz __read_mostly;
+static pserialize_t key_sad_psz __read_mostly;
/* SPD */
static struct {
@@ -220,6 +243,7 @@ static struct {
/* SAD */
static struct {
kmutex_t lock;
+ kcondvar_t cv;
struct pslist_head sahlist;
} key_sad __cacheline_aligned;
@@ -614,13 +638,18 @@ static struct mbuf *key_setdumpsp (struc
static u_int key_getspreqmsglen (const struct secpolicy *);
static int key_spdexpire (struct secpolicy *);
static struct secashead *key_newsah (const struct secasindex *);
-static void key_delsah (struct secashead *);
+static void key_unlink_sah(struct secashead *);
+static void key_destroy_sah(struct secashead *);
+static bool key_sah_has_sav(struct secashead *);
+static void key_sah_ref(struct secashead *);
+static void key_sah_unref(struct secashead *);
static struct secasvar *key_newsav(struct mbuf *,
const struct sadb_msghdr *, int *, const char*, int);
#define KEY_NEWSAV(m, sadb, e) \
key_newsav(m, sadb, e, __func__, __LINE__)
static void key_delsav (struct secasvar *);
static struct secashead *key_getsah(const struct secasindex *, int);
+static struct secashead *key_getsah_ref(const struct secasindex *, int);
static bool key_checkspidup(const struct secasindex *, u_int32_t);
static struct secasvar *key_getsavbyspi (struct secashead *, u_int32_t);
static int key_setsaval (struct secasvar *, struct mbuf *,
@@ -800,7 +829,7 @@ key_unlink_sp(struct secpolicy *sp)
#ifdef NET_MPSAFE
KASSERT(mutex_ownable(softnet_lock));
- pserialize_perform(key_psz);
+ pserialize_perform(key_spd_psz);
#endif
localcount_drain(&sp->localcount, &key_spd.cv, &key_spd.lock);
@@ -2954,9 +2983,13 @@ key_newsah(const struct secasindex *said
PSLIST_INIT(&newsah->savlist[i]);
newsah->saidx = *saidx;
- /* add to saidxtree */
- newsah->state = SADB_SASTATE_MATURE;
+ localcount_init(&newsah->localcount);
+ /* Take a reference for the caller */
+ localcount_acquire(&newsah->localcount);
+
+ /* Add to the sah list */
SAHLIST_ENTRY_INIT(newsah);
+ newsah->state = SADB_SASTATE_MATURE;
mutex_enter(&key_sad.lock);
SAHLIST_WRITER_INSERT_HEAD(newsah);
mutex_exit(&key_sad.lock);
@@ -2964,51 +2997,55 @@ key_newsah(const struct secasindex *said
return newsah;
}
-/*
- * delete SA index and all SA registerd.
- */
-static void
-key_delsah(struct secashead *sah)
+static bool
+key_sah_has_sav(struct secashead *sah)
{
- struct secasvar *sav;
u_int state;
- int s;
- int zombie = 0;
- KASSERT(!cpu_softintr_p());
- KASSERT(sah != NULL);
-
- s = splsoftnet();
+ KASSERT(mutex_owned(&key_sad.lock));
- /* searching all SA registerd in the secindex. */
SASTATE_ANY_FOREACH(state) {
- SAVLIST_READER_FOREACH(sav, sah, state) {
- /* give up to delete this sa */
- zombie++;
- }
+ if (!SAVLIST_WRITER_EMPTY(sah, state))
+ return true;
}
- /* don't delete sah only if there are savs. */
- if (zombie) {
- splx(s);
- return;
- }
+ return false;
+}
- rtcache_free(&sah->sa_route);
+static void
+key_unlink_sah(struct secashead *sah)
+{
- /* remove from tree of SA index */
+ KASSERT(!cpu_softintr_p());
+ KASSERT(mutex_owned(&key_sad.lock));
+ KASSERT(sah->state == SADB_SASTATE_DEAD);
+
+ /* Remove from the sah list */
SAHLIST_WRITER_REMOVE(sah);
+#ifdef NET_MPSAFE
+ KASSERT(mutex_ownable(softnet_lock));
+ pserialize_perform(key_sad_psz);
+#endif
+
+ localcount_drain(&sah->localcount, &key_sad.cv, &key_sad.lock);
+}
+
+static void
+key_destroy_sah(struct secashead *sah)
+{
+
+ rtcache_free(&sah->sa_route);
+
+ SAHLIST_ENTRY_DESTROY(sah);
+ localcount_fini(&sah->localcount);
+
if (sah->idents != NULL)
kmem_free(sah->idents, sah->idents_len);
if (sah->identd != NULL)
kmem_free(sah->identd, sah->identd_len);
- SAHLIST_ENTRY_DESTROY(sah);
kmem_free(sah, sizeof(*sah));
-
- splx(s);
- return;
}
/*
@@ -3136,7 +3173,32 @@ key_delsav(struct secasvar *sav)
}
/*
- * search SAD.
+ * Must be called in a pserialize read section. A held sah
+ * must be released by key_sah_unref after use.
+ */
+static void
+key_sah_ref(struct secashead *sah)
+{
+
+ localcount_acquire(&sah->localcount);
+}
+
+/*
+ * Must be called without holding key_sad.lock because the lock
+ * would be held in localcount_release.
+ */
+static void
+key_sah_unref(struct secashead *sah)
+{
+
+ KDASSERT(mutex_ownable(&key_sad.lock));
+
+ localcount_release(&sah->localcount, &key_sad.cv, &key_sad.lock);
+}
+
+/*
+ * Search SAD and return sah. Must be called in a pserialize
+ * read section.
* OUT:
* NULL : not found
* others : found, pointer to a SA.
@@ -3157,6 +3219,28 @@ key_getsah(const struct secasindex *said
}
/*
+ * Search SAD and return sah. If sah is returned, the caller must call
+ * key_sah_unref to releaset a reference.
+ * OUT:
+ * NULL : not found
+ * others : found, pointer to a SA.
+ */
+static struct secashead *
+key_getsah_ref(const struct secasindex *saidx, int flag)
+{
+ struct secashead *sah;
+ int s;
+
+ s = pserialize_read_enter();
+ sah = key_getsah(saidx, flag);
+ if (sah != NULL)
+ key_sah_ref(sah);
+ pserialize_read_exit(s);
+
+ return sah;
+}
+
+/*
* check not to be duplicated SPI.
* NOTE: this function is too slow due to searching all SAD.
* OUT:
@@ -3168,6 +3252,7 @@ key_checkspidup(const struct secasindex
{
struct secashead *sah;
struct secasvar *sav;
+ int s;
/* check address family */
if (saidx->src.sa.sa_family != saidx->dst.sa.sa_family) {
@@ -3176,15 +3261,18 @@ key_checkspidup(const struct secasindex
}
/* check all SAD */
+ s = pserialize_read_enter();
SAHLIST_READER_FOREACH(sah) {
if (!key_ismyaddr((struct sockaddr *)&sah->saidx.dst))
continue;
sav = key_getsavbyspi(sah, spi);
if (sav != NULL) {
+ pserialize_read_exit(s);
KEY_SA_UNREF(&sav);
return true;
}
}
+ pserialize_read_exit(s);
return false;
}
@@ -4594,15 +4682,28 @@ static void
key_timehandler_sad(time_t now)
{
struct secashead *sah;
- struct secasvar *sav;
+ int s;
restart:
+ mutex_enter(&key_sad.lock);
SAHLIST_WRITER_FOREACH(sah) {
- /* if sah has been dead, then delete it and process next sah. */
- if (sah->state == SADB_SASTATE_DEAD) {
- key_delsah(sah);
+ /* If sah has been dead and has no sav, then delete it */
+ if (sah->state == SADB_SASTATE_DEAD &&
+ !key_sah_has_sav(sah)) {
+ key_unlink_sah(sah);
+ mutex_exit(&key_sad.lock);
+ key_destroy_sah(sah);
goto restart;
}
+ }
+ mutex_exit(&key_sad.lock);
+
+ s = pserialize_read_enter();
+ SAHLIST_READER_FOREACH(sah) {
+ struct secasvar *sav;
+
+ key_sah_ref(sah);
+ pserialize_read_exit(s);
/* if LARVAL entry doesn't become MATURE, delete it. */
restart_sav_LARVAL:
@@ -4727,7 +4828,11 @@ restart:
* (such as from SPD).
*/
}
+
+ s = pserialize_read_enter();
+ key_sah_unref(sah);
}
+ pserialize_read_exit(s);
}
static void
@@ -4984,7 +5089,7 @@ key_api_getspi(struct socket *so, struct
return key_senderror(so, m, EINVAL);
/* get a SA index */
- sah = key_getsah(&saidx, CMP_REQID);
+ sah = key_getsah_ref(&saidx, CMP_REQID);
if (sah == NULL) {
/* create a new SA index */
sah = key_newsah(&saidx);
@@ -4998,6 +5103,7 @@ key_api_getspi(struct socket *so, struct
/* XXX rewrite */
newsav = KEY_NEWSAV(m, mhp, &error);
if (newsav == NULL) {
+ key_sah_unref(sah);
/* XXX don't free new SA index allocated in above. */
return key_senderror(so, m, error);
}
@@ -5015,6 +5121,8 @@ key_api_getspi(struct socket *so, struct
mutex_exit(&key_sad.lock);
key_validate_savlist(sah, SADB_SASTATE_LARVAL);
+ key_sah_unref(sah);
+
#ifndef IPSEC_NONBLOCK_ACQUIRE
/* delete the entry in key_misc.acqlist */
if (mhp->msg->sadb_msg_seq != 0) {
@@ -5362,7 +5470,7 @@ key_api_update(struct socket *so, struct
return key_senderror(so, m, EINVAL);
/* get a SA header */
- sah = key_getsah(&saidx, CMP_REQID);
+ sah = key_getsah_ref(&saidx, CMP_REQID);
if (sah == NULL) {
IPSECLOG(LOG_DEBUG, "no SA index found.\n");
return key_senderror(so, m, ENOENT);
@@ -5372,7 +5480,7 @@ key_api_update(struct socket *so, struct
/* XXX rewrite */
error = key_setident(sah, m, mhp);
if (error)
- return key_senderror(so, m, error);
+ goto error_sah;
/* find a SA with sequence number. */
#ifdef IPSEC_DOSEQCHECK
@@ -5382,7 +5490,8 @@ key_api_update(struct socket *so, struct
IPSECLOG(LOG_DEBUG,
"no larval SA with sequence %u exists.\n",
mhp->msg->sadb_msg_seq);
- return key_senderror(so, m, ENOENT);
+ error = ENOENT;
+ goto error_sah;
}
}
#else
@@ -5390,7 +5499,8 @@ key_api_update(struct socket *so, struct
if (sav == NULL) {
IPSECLOG(LOG_DEBUG, "no such a SA found (spi:%u)\n",
(u_int32_t)ntohl(sa0->sadb_sa_spi));
- return key_senderror(so, m, EINVAL);
+ error = EINVAL;
+ goto error_sah;
}
#endif
@@ -5461,6 +5571,9 @@ key_api_update(struct socket *so, struct
KEY_FREESAV(&sav);
KEY_FREESAV(&sav);
+ key_sah_unref(sah);
+ sah = NULL;
+
{
struct mbuf *n;
@@ -5476,6 +5589,8 @@ key_api_update(struct socket *so, struct
}
error:
KEY_SA_UNREF(&sav);
+error_sah:
+ key_sah_unref(sah);
return key_senderror(so, m, error);
}
@@ -5593,7 +5708,7 @@ key_api_add(struct socket *so, struct mb
return key_senderror(so, m, EINVAL);
/* get a SA header */
- sah = key_getsah(&saidx, CMP_REQID);
+ sah = key_getsah_ref(&saidx, CMP_REQID);
if (sah == NULL) {
/* create a new SA header */
sah = key_newsah(&saidx);
@@ -5606,9 +5721,8 @@ key_api_add(struct socket *so, struct mb
/* set spidx if there */
/* XXX rewrite */
error = key_setident(sah, m, mhp);
- if (error) {
- return key_senderror(so, m, error);
- }
+ if (error)
+ goto error;
{
struct secasvar *sav;
@@ -5618,27 +5732,28 @@ key_api_add(struct socket *so, struct mb
if (sav != NULL) {
KEY_SA_UNREF(&sav);
IPSECLOG(LOG_DEBUG, "SA already exists.\n");
- return key_senderror(so, m, EEXIST);
+ error = EEXIST;
+ goto error;
}
}
/* create new SA entry. */
newsav = KEY_NEWSAV(m, mhp, &error);
- if (newsav == NULL) {
- return key_senderror(so, m, error);
- }
+ if (newsav == NULL)
+ goto error;
newsav->sah = sah;
error = key_handle_natt_info(newsav, mhp);
if (error != 0) {
key_delsav(newsav);
- return key_senderror(so, m, EINVAL);
+ error = EINVAL;
+ goto error;
}
error = key_init_xform(newsav);
if (error != 0) {
key_delsav(newsav);
- return key_senderror(so, m, error);
+ goto error;
}
/* add to satree */
@@ -5650,6 +5765,9 @@ key_api_add(struct socket *so, struct mb
mutex_exit(&key_sad.lock);
key_validate_savlist(sah, SADB_SASTATE_MATURE);
+ key_sah_unref(sah);
+ sah = NULL;
+
/*
* don't call key_freesav() here, as we would like to keep the SA
* in the database on success.
@@ -5668,6 +5786,9 @@ key_api_add(struct socket *so, struct mb
m_freem(m);
return key_sendup_mbuf(so, n, KEY_SENDUP_ALL);
}
+error:
+ key_sah_unref(sah);
+ return key_senderror(so, m, error);
}
/* m is retained */
@@ -5853,10 +5974,11 @@ key_api_delete(struct socket *so, struct
return key_senderror(so, m, EINVAL);
/* get a SA header */
- sah = key_getsah(&saidx, CMP_HEAD);
+ sah = key_getsah_ref(&saidx, CMP_HEAD);
if (sah != NULL) {
/* get a SA with SPI. */
sav = key_getsavbyspi(sah, sa0->sadb_sa_spi);
+ key_sah_unref(sah);
}
if (sav == NULL) {
@@ -5911,7 +6033,7 @@ key_delete_all(struct socket *so, struct
if (error != 0)
return key_senderror(so, m, EINVAL);
- sah = key_getsah(&saidx, CMP_HEAD);
+ sah = key_getsah_ref(&saidx, CMP_HEAD);
if (sah != NULL) {
/* Delete all non-LARVAL SAs. */
SASTATE_ALIVE_FOREACH(state) {
@@ -5933,6 +6055,7 @@ key_delete_all(struct socket *so, struct
goto restart;
}
}
+ key_sah_unref(sah);
}
{
struct mbuf *n;
@@ -5971,7 +6094,6 @@ key_api_get(struct socket *so, struct mb
struct sadb_sa *sa0;
const struct sockaddr *src, *dst;
struct secasindex saidx;
- struct secashead *sah;
struct secasvar *sav = NULL;
u_int16_t proto;
int error;
@@ -6008,11 +6130,17 @@ key_api_get(struct socket *so, struct mb
return key_senderror(so, m, EINVAL);
/* get a SA header */
+ {
+ struct secashead *sah;
+ int s = pserialize_read_enter();
+
sah = key_getsah(&saidx, CMP_HEAD);
if (sah != NULL) {
/* get a SA with SPI. */
sav = key_getsavbyspi(sah, sa0->sadb_sa_spi);
}
+ pserialize_read_exit(s);
+ }
if (sav == NULL) {
IPSECLOG(LOG_DEBUG, "no SA found.\n");
return key_senderror(so, m, ENOENT);
@@ -6023,7 +6151,7 @@ key_api_get(struct socket *so, struct mb
u_int8_t satype;
/* map proto to satype */
- satype = key_proto2satype(sah->saidx.proto);
+ satype = key_proto2satype(sav->sah->saidx.proto);
if (satype == 0) {
KEY_SA_UNREF(&sav);
IPSECLOG(LOG_DEBUG, "there was invalid proto in SAD.\n");
@@ -6623,7 +6751,6 @@ key_api_acquire(struct socket *so, struc
{
const struct sockaddr *src, *dst;
struct secasindex saidx;
- struct secashead *sah;
u_int16_t proto;
int error;
@@ -6703,11 +6830,18 @@ key_api_acquire(struct socket *so, struc
return key_senderror(so, m, EINVAL);
/* get a SA index */
+ {
+ struct secashead *sah;
+ int s = pserialize_read_enter();
+
sah = key_getsah(&saidx, CMP_MODE_REQID);
if (sah != NULL) {
+ pserialize_read_exit(s);
IPSECLOG(LOG_DEBUG, "a SA exists already.\n");
return key_senderror(so, m, EEXIST);
}
+ pserialize_read_exit(s);
+ }
error = key_acquire(&saidx, NULL);
if (error != 0) {
@@ -7051,6 +7185,7 @@ key_api_flush(struct socket *so, struct
struct secasvar *sav;
u_int16_t proto;
u_int8_t state;
+ int s;
/* map satype to proto */
proto = key_satype2proto(mhp->msg->sadb_msg_satype);
@@ -7060,11 +7195,15 @@ key_api_flush(struct socket *so, struct
}
/* no SATYPE specified, i.e. flushing all SA. */
+ s = pserialize_read_enter();
SAHLIST_READER_FOREACH(sah) {
if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC &&
proto != sah->saidx.proto)
continue;
+ key_sah_ref(sah);
+ pserialize_read_exit(s);
+
SASTATE_ALIVE_FOREACH(state) {
restart:
SAVLIST_WRITER_FOREACH(sav, sah, state) {
@@ -7074,8 +7213,11 @@ key_api_flush(struct socket *so, struct
}
}
+ s = pserialize_read_enter();
sah->state = SADB_SASTATE_DEAD;
+ key_sah_unref(sah);
}
+ pserialize_read_exit(s);
if (m->m_len < sizeof(struct sadb_msg) ||
sizeof(struct sadb_msg) > m->m_len + M_TRAILINGSPACE(m)) {
@@ -7784,10 +7926,12 @@ key_do_init(void)
int i, error;
mutex_init(&key_misc.lock, MUTEX_DEFAULT, IPL_NONE);
- key_psz = pserialize_create();
+ key_spd_psz = pserialize_create();
mutex_init(&key_spd.lock, MUTEX_DEFAULT, IPL_NONE);
cv_init(&key_spd.cv, "key_sp");
+ key_sad_psz = pserialize_create();
mutex_init(&key_sad.lock, MUTEX_DEFAULT, IPL_NONE);
+ cv_init(&key_sad.cv, "key_sa");
pfkeystat_percpu = percpu_alloc(sizeof(uint64_t) * PFKEY_NSTATS);
@@ -7982,16 +8126,26 @@ void
key_sa_routechange(struct sockaddr *dst)
{
struct secashead *sah;
- struct route *ro;
- const struct sockaddr *sa;
+ int s;
+ s = pserialize_read_enter();
SAHLIST_READER_FOREACH(sah) {
+ struct route *ro;
+ const struct sockaddr *sa;
+
+ key_sah_ref(sah);
+ pserialize_read_exit(s);
+
ro = &sah->sa_route;
sa = rtcache_getdst(ro);
if (sa != NULL && dst->sa_len == sa->sa_len &&
memcmp(dst, sa, dst->sa_len) == 0)
rtcache_free(ro);
+
+ s = pserialize_read_enter();
+ key_sah_unref(sah);
}
+ pserialize_read_exit(s);
return;
}
Index: src/sys/netipsec/keydb.h
diff -u src/sys/netipsec/keydb.h:1.18 src/sys/netipsec/keydb.h:1.19
--- src/sys/netipsec/keydb.h:1.18 Mon Aug 7 03:21:59 2017
+++ src/sys/netipsec/keydb.h Tue Aug 8 04:17:34 2017
@@ -1,4 +1,4 @@
-/* $NetBSD: keydb.h,v 1.18 2017/08/07 03:21:59 ozaki-r Exp $ */
+/* $NetBSD: keydb.h,v 1.19 2017/08/08 04:17:34 ozaki-r Exp $ */
/* $FreeBSD: src/sys/netipsec/keydb.h,v 1.1.4.1 2003/01/24 05:11:36 sam Exp $ */
/* $KAME: keydb.h,v 1.14 2000/08/02 17:58:26 sakane Exp $ */
@@ -36,6 +36,8 @@
#ifdef _KERNEL
+#include <sys/localcount.h>
+
#include <netipsec/key_var.h>
#include <net/route.h>
#include <netinet/in.h>
@@ -66,6 +68,7 @@ struct secasindex {
/* Security Association Data Base */
struct secashead {
struct pslist_entry pslist_entry;
+ struct localcount localcount; /* reference count */
struct secasindex saidx;
