Module Name: src
Committed By: ozaki-r
Date: Tue Feb 20 03:34:52 UTC 2018
Modified Files:
src/sys/kern: subr_lockdebug.c
Log Message:
Spinkle __predict_false to LOCKDEBUG functions
Panics and lockdebug failures are unlikely to occur normally.
To generate a diff of this commit:
cvs rdiff -u -r1.59 -r1.60 src/sys/kern/subr_lockdebug.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/subr_lockdebug.c
diff -u src/sys/kern/subr_lockdebug.c:1.59 src/sys/kern/subr_lockdebug.c:1.60
--- src/sys/kern/subr_lockdebug.c:1.59 Wed Feb 14 03:56:26 2018
+++ src/sys/kern/subr_lockdebug.c Tue Feb 20 03:34:52 2018
@@ -1,4 +1,4 @@
-/* $NetBSD: subr_lockdebug.c,v 1.59 2018/02/14 03:56:26 ozaki-r Exp $ */
+/* $NetBSD: subr_lockdebug.c,v 1.60 2018/02/20 03:34:52 ozaki-r Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -34,7 +34,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.59 2018/02/14 03:56:26 ozaki-r Exp $");
+__KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.60 2018/02/20 03:34:52 ozaki-r Exp $");
#ifdef _KERNEL_OPT
#include "opt_ddb.h"
@@ -196,7 +196,7 @@ lockdebug_lookup(const char *func, size_
lockdebug_t *ld;
ld = lockdebug_lookup1(lock);
- if (ld == NULL) {
+ if (__predict_false(ld == NULL)) {
panic("%s,%zu: uninitialized lock (lock=%p, from=%08"
PRIxPTR ")", func, line, lock, where);
}
@@ -246,14 +246,14 @@ lockdebug_alloc(const char *func, size_t
lockdebug_t *ld;
int s;
- if (lo == NULL || panicstr != NULL || ld_panic)
+ if (__predict_false(lo == NULL || panicstr != NULL || ld_panic))
return false;
- if (ld_freeptr == 0)
+ if (__predict_false(ld_freeptr == 0))
lockdebug_init();
s = splhigh();
__cpu_simple_lock(&ld_mod_lk);
- if ((ld = lockdebug_lookup1(lock)) != NULL) {
+ if (__predict_false((ld = lockdebug_lookup1(lock)) != NULL)) {
__cpu_simple_unlock(&ld_mod_lk);
lockdebug_abort1(func, line, ld, s, "already initialized",
true);
@@ -281,7 +281,7 @@ lockdebug_alloc(const char *func, size_t
} else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) {
s = lockdebug_more(s);
}
- if ((ld = TAILQ_FIRST(&ld_free)) == NULL) {
+ if (__predict_false((ld = TAILQ_FIRST(&ld_free)) == NULL)) {
__cpu_simple_unlock(&ld_mod_lk);
splx(s);
return false;
@@ -290,7 +290,7 @@ lockdebug_alloc(const char *func, size_t
ld_nfree--;
ci->ci_lkdebug_recurse--;
- if (ld->ld_lock != NULL) {
+ if (__predict_false(ld->ld_lock != NULL)) {
panic("%s,%zu: corrupt table ld %p", func, line, ld);
}
@@ -322,20 +322,21 @@ lockdebug_free(const char *func, size_t
lockdebug_t *ld;
int s;
- if (panicstr != NULL || ld_panic)
+ if (__predict_false(panicstr != NULL || ld_panic))
return;
s = splhigh();
__cpu_simple_lock(&ld_mod_lk);
ld = lockdebug_lookup(func, line, lock,
(uintptr_t) __builtin_return_address(0));
- if (ld == NULL) {
+ if (__predict_false(ld == NULL)) {
__cpu_simple_unlock(&ld_mod_lk);
panic("%s,%zu: destroying uninitialized object %p"
"(ld_lock=%p)", func, line, lock, ld->ld_lock);
return;
}
- if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {
+ if (__predict_false((ld->ld_flags & LD_LOCKED) != 0 ||
+ ld->ld_shares != 0)) {
__cpu_simple_unlock(&ld_mod_lk);
lockdebug_abort1(func, line, ld, s, "is locked or in use",
true);
@@ -430,7 +431,7 @@ lockdebug_wantlock(const char *func, siz
(void)shared;
recurse = false;
- if (panicstr != NULL || ld_panic)
+ if (__predict_false(panicstr != NULL || ld_panic))
return;
s = splhigh();
@@ -446,7 +447,7 @@ lockdebug_wantlock(const char *func, siz
recurse = true;
}
if (cpu_intr_p()) {
- if ((ld->ld_flags & LD_SLEEPER) != 0) {
+ if (__predict_false((ld->ld_flags & LD_SLEEPER) != 0)) {
lockdebug_abort1(func, line, ld, s,
"acquiring sleep lock from interrupt context",
true);
@@ -457,7 +458,7 @@ lockdebug_wantlock(const char *func, siz
ld->ld_shwant++;
else if (shared == 0)
ld->ld_exwant++;
- if (recurse) {
+ if (__predict_false(recurse)) {
lockdebug_abort1(func, line, ld, s, "locking against myself",
true);
return;
@@ -479,7 +480,7 @@ lockdebug_locked(const char *func, size_
lockdebug_t *ld;
int s;
- if (panicstr != NULL || ld_panic)
+ if (__predict_false(panicstr != NULL || ld_panic))
return;
s = splhigh();
@@ -493,7 +494,7 @@ lockdebug_locked(const char *func, size_
/* nothing */
} else if (ld->ld_shares++ == 0) {
ld->ld_locked = (uintptr_t)cvlock;
- } else if (cvlock != (void *)ld->ld_locked) {
+ } else if (__predict_false(cvlock != (void *)ld->ld_locked)) {
lockdebug_abort1(func, line, ld, s,
"multiple locks used with condition variable",
true);
@@ -505,7 +506,7 @@ lockdebug_locked(const char *func, size_
ld->ld_shares++;
ld->ld_shwant--;
} else {
- if ((ld->ld_flags & LD_LOCKED) != 0) {
+ if (__predict_false((ld->ld_flags & LD_LOCKED) != 0)) {
lockdebug_abort1(func, line, ld, s, "already locked",
true);
return;
@@ -539,7 +540,7 @@ lockdebug_unlocked(const char *func, siz
lockdebug_t *ld;
int s;
- if (panicstr != NULL || ld_panic)
+ if (__predict_false(panicstr != NULL || ld_panic))
return;
s = splhigh();
@@ -554,12 +555,12 @@ lockdebug_unlocked(const char *func, siz
ld->ld_shares--;
}
} else if (shared) {
- if (l->l_shlocks == 0) {
+ if (__predict_false(l->l_shlocks == 0)) {
lockdebug_abort1(func, line, ld, s,
"no shared locks held by LWP", true);
return;
}
- if (ld->ld_shares == 0) {
+ if (__predict_false(ld->ld_shares == 0)) {
lockdebug_abort1(func, line, ld, s,
"no shared holds on this lock", true);
return;
@@ -573,20 +574,21 @@ lockdebug_unlocked(const char *func, siz
if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))
ld->ld_cpu = (uint16_t)-1;
} else {
- if ((ld->ld_flags & LD_LOCKED) == 0) {
+ if (__predict_false((ld->ld_flags & LD_LOCKED) == 0)) {
lockdebug_abort1(func, line, ld, s, "not locked", true);
return;
}
if ((ld->ld_flags & LD_SLEEPER) != 0) {
- if (ld->ld_lwp != curlwp) {
+ if (__predict_false(ld->ld_lwp != curlwp)) {
lockdebug_abort1(func, line, ld, s,
"not held by current LWP", true);
return;
}
TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain);
} else {
- if (ld->ld_cpu != (uint16_t)cpu_index(curcpu())) {
+ uint16_t idx = (uint16_t)cpu_index(curcpu());
+ if (__predict_false(ld->ld_cpu != idx)) {
lockdebug_abort1(func, line, ld, s,
"not held by current CPU", true);
return;
@@ -614,7 +616,7 @@ lockdebug_wakeup(const char *func, size_
lockdebug_t *ld;
int s;
- if (panicstr != NULL || ld_panic || lock == (void *)&lbolt)
+ if (__predict_false(panicstr != NULL || ld_panic || lock == (void *)&lbolt))
return;
s = splhigh();
@@ -627,7 +629,8 @@ lockdebug_wakeup(const char *func, size_
* If it has any waiters, ensure that they are using the
* same interlock.
*/
- if (ld->ld_shares != 0 && !mutex_owned((kmutex_t *)ld->ld_locked)) {
+ if (__predict_false(ld->ld_shares != 0 &&
+ !mutex_owned((kmutex_t *)ld->ld_locked))) {
lockdebug_abort1(func, line, ld, s, "interlocking mutex not "
"held during wakeup", true);
return;
@@ -650,7 +653,7 @@ lockdebug_barrier(const char *func, size
lockdebug_t *ld;
int s;
- if (panicstr != NULL || ld_panic)
+ if (__predict_false(panicstr != NULL || ld_panic))
return;
s = splhigh();
@@ -669,7 +672,8 @@ lockdebug_barrier(const char *func, size
splx(s);
return;
}
- if ((ld = TAILQ_FIRST(&l->l_ld_locks)) != NULL) {
+ ld = TAILQ_FIRST(&l->l_ld_locks);
+ if (__predict_false(ld != NULL)) {
__cpu_simple_lock(&ld->ld_spinlock);
lockdebug_abort1(func, line, ld, s, "sleep lock held", true);
return;
@@ -700,7 +704,7 @@ lockdebug_mem_check(const char *func, si
struct cpu_info *ci;
int s;
- if (panicstr != NULL || ld_panic)
+ if (__predict_false(panicstr != NULL || ld_panic))
return;
s = splhigh();
@@ -710,14 +714,14 @@ lockdebug_mem_check(const char *func, si
if (ld != NULL) {
const uintptr_t lock = (uintptr_t)ld->ld_lock;
- if ((uintptr_t)base > lock)
+ if (__predict_false((uintptr_t)base > lock))
panic("%s,%zu: corrupt tree ld=%p, base=%p, sz=%zu",
func, line, ld, base, sz);
if (lock >= (uintptr_t)base + sz)
ld = NULL;
}
__cpu_simple_unlock(&ci->ci_data.cpu_ld_lock);
- if (ld != NULL) {
+ if (__predict_false(ld != NULL)) {
__cpu_simple_lock(&ld->ld_spinlock);
lockdebug_abort1(func, line, ld, s,
"allocation contains active lock", !cold);
