This patch enables the collection of event counts in the slowpath of the
realtime queued spinlocks. The following events are being tracked if
CONFIG_QUEUED_LOCK_STAT is defined:
- # of interrupt context RT spinnings
- # of task context RT spinnings
- # of nested spinlock RT spinnings
- # of unqueue operations due to RT priority
- # of unqueue operations due to need_resched()
- # of CPU yieldings
Signed-off-by: Waiman Long <long...@redhat.com>
---
arch/x86/Kconfig | 16 ++++----
kernel/locking/qspinlock_rt.h | 15 +++++++-
kernel/locking/qspinlock_stat.h | 81 ++++++++++++++++++++++++++++++++++++-----
3 files changed, 92 insertions(+), 20 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 7a97b31..e0dc3c8 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -694,6 +694,14 @@ config SCHED_OMIT_FRAME_POINTER
If in doubt, say "Y".
+config QUEUED_LOCK_STAT
+ bool "Queued spinlock statistics"
+ depends on (PARAVIRT_SPINLOCKS || REALTIME_QUEUED_SPINLOCKS) && DEBUG_FS
+ ---help---
+ Enable the collection of statistical data on the slowpath
+ behavior of paravirtualized or realtime queued spinlocks
+ and report them on debugfs.
+
menuconfig HYPERVISOR_GUEST
bool "Linux guest support"
---help---
@@ -734,14 +742,6 @@ config PARAVIRT_SPINLOCKS
If you are unsure how to answer this question, answer Y.
-config QUEUED_LOCK_STAT
- bool "Paravirt queued spinlock statistics"
- depends on PARAVIRT_SPINLOCKS && DEBUG_FS
- ---help---
- Enable the collection of statistical data on the slowpath
- behavior of paravirtualized queued spinlocks and report
- them on debugfs.
-
source "arch/x86/xen/Kconfig"
config KVM_GUEST
diff --git a/kernel/locking/qspinlock_rt.h b/kernel/locking/qspinlock_rt.h
index 18ec1f8..f6f8498 100644
--- a/kernel/locking/qspinlock_rt.h
+++ b/kernel/locking/qspinlock_rt.h
@@ -55,6 +55,7 @@
* inner lock, finish up its work, release the locks and reenable preemption.
*/
#include <linux/sched.h>
+#include "qspinlock_stat.h"
#ifndef MAX
#define MAX(a, b) (((a) >= (b)) ? (a) : (b))
@@ -157,6 +158,7 @@ static bool __rt_spin_trylock(struct qspinlock *lock,
if (!(prio = rt_task_priority(task, min_prio)))
return false;
+ qstat_inc_either(qstat_rt_spin_task, qstat_rt_spin_irq, task);
/*
* Spin on the lock and try to set its priority into the pending byte.
@@ -237,6 +239,7 @@ static bool __rt_spin_trylock(struct qspinlock *lock,
if (pdprio == mypdprio)
cmpxchg_relaxed(&l->pending, pdprio, 0);
}
+ qstat_inc(qstat_rt_resched, true);
schedule_preempt_disabled();
}
return true;
@@ -349,6 +352,9 @@ static bool rt_wait_node_or_unqueue(struct qspinlock *lock,
return false;
unqueue:
+ qstat_inc_either(qstat_rt_unqueue_sched, qstat_rt_unqueue_prio,
+ need_resched());
+
/*
* Step - A -- stabilize @prev
*
@@ -406,8 +412,10 @@ static bool rt_wait_node_or_unqueue(struct qspinlock *lock,
/*
* Yield the CPU if needed by another task with the right condition.
*/
- if (rt_should_resched())
+ if (rt_should_resched()) {
+ qstat_inc(qstat_rt_resched, true);
schedule_preempt_disabled();
+ }
return true; /* Need to retry RT spinning */
}
@@ -486,8 +494,10 @@ static u32 rt_spin_lock_or_retry(struct qspinlock *lock,
/*
* Yield the CPU if needed by another task with the right condition.
*/
- if (retry && rt_should_resched())
+ if (retry && rt_should_resched()) {
+ qstat_inc(qstat_rt_resched, true);
schedule_preempt_disabled();
+ }
return retry ? RT_RETRY : 1;
}
@@ -514,6 +524,7 @@ void __lockfunc _rt_raw_spin_lock_nested(raw_spinlock_t
*lock, int subclass,
_RET_IP_);
}
#endif
+ qstat_inc(qstat_rt_spin_nest, true);
__acquire(lock);
__rt_spin_trylock(&lock->raw_lock,
outerlock ? &outerlock->raw_lock : NULL, 1);
diff --git a/kernel/locking/qspinlock_stat.h b/kernel/locking/qspinlock_stat.h
index e852be4..d212c71 100644
--- a/kernel/locking/qspinlock_stat.h
+++ b/kernel/locking/qspinlock_stat.h
@@ -14,9 +14,10 @@
/*
* When queued spinlock statistical counters are enabled, the following
- * debugfs files will be created for reporting the counter values:
+ * debugfs files will be created under the <debugfs>/qlockstat directory
+ * for reporting the counter values:
*
- * <debugfs>/qlockstat/
+ * PV qspinlock specific files:
* pv_hash_hops - average # of hops per hashing operation
* pv_kick_unlock - # of vCPU kicks issued at unlock time
* pv_kick_wake - # of vCPU kicks used for computing pv_latency_wake
@@ -30,6 +31,14 @@
* pv_wait_head - # of vCPU wait's at the queue head
* pv_wait_node - # of vCPU wait's at a non-head queue node
*
+ * RT qspinlock specific files:
+ * rt_resched - # of voluntary CPU yieldings
+ * rt_spin_irq - # of interrupt context RT spinnings
+ * rt_spin_nest - # of nested spinlock RT spinnings
+ * rt_spin_task - # of task context RT spinnings
+ * rt_unqueue_prio - # of unqueue operations due to RT priority
+ * rt_unqueue_sched - # of unqueue operations due to need_resched()
+ *
* Writing to the "reset_counters" file will reset all the above counter
* values.
*
@@ -40,12 +49,11 @@
*
* There may be slight difference between pv_kick_wake and pv_kick_unlock.
*/
+#ifndef __KERNEL_LOCKING_QSPINLOCK_STAT_H
+#define __KERNEL_LOCKING_QSPINLOCK_STAT_H
+
enum qlock_stats {
- qstat_pv_hash_hops,
- qstat_pv_kick_unlock,
- qstat_pv_kick_wake,
- qstat_pv_latency_kick,
- qstat_pv_latency_wake,
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
qstat_pv_lock_slowpath,
qstat_pv_lock_stealing,
qstat_pv_spurious_wakeup,
@@ -53,6 +61,26 @@ enum qlock_stats {
qstat_pv_wait_early,
qstat_pv_wait_head,
qstat_pv_wait_node,
+#endif
+ /*
+ * These enums are needed to avoid compilation error even though
+ * they are not used when CONFIG_PARAVIRT_SPINLOCKS isn't defined.
+ */
+ qstat_pv_hash_hops,
+ qstat_pv_latency_kick,
+ qstat_pv_latency_wake,
+ qstat_pv_kick_unlock,
+ qstat_pv_kick_wake,
+
+#ifdef CONFIG_REALTIME_QUEUED_SPINLOCKS
+ qstat_rt_resched,
+ qstat_rt_spin_irq,
+ qstat_rt_spin_nest,
+ qstat_rt_spin_task,
+ qstat_rt_unqueue_prio,
+ qstat_rt_unqueue_sched,
+#endif
+
qstat_num, /* Total number of statistical counters */
qstat_reset_cnts = qstat_num,
};
@@ -66,6 +94,7 @@ enum qlock_stats {
#include <linux/fs.h>
static const char * const qstat_names[qstat_num + 1] = {
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
[qstat_pv_hash_hops] = "pv_hash_hops",
[qstat_pv_kick_unlock] = "pv_kick_unlock",
[qstat_pv_kick_wake] = "pv_kick_wake",
@@ -78,6 +107,17 @@ enum qlock_stats {
[qstat_pv_wait_early] = "pv_wait_early",
[qstat_pv_wait_head] = "pv_wait_head",
[qstat_pv_wait_node] = "pv_wait_node",
+#endif
+
+#ifdef CONFIG_REALTIME_QUEUED_SPINLOCKS
+ [qstat_rt_resched] = "rt_resched",
+ [qstat_rt_spin_irq] = "rt_spin_irq",
+ [qstat_rt_spin_nest] = "rt_spin_nest",
+ [qstat_rt_spin_task] = "rt_spin_task",
+ [qstat_rt_unqueue_prio] = "rt_unqueue_prio",
+ [qstat_rt_unqueue_sched] = "rt_unqueue_sched",
+#endif
+
[qstat_reset_cnts] = "reset_counters",
};
@@ -85,7 +125,9 @@ enum qlock_stats {
* Per-cpu counters
*/
static DEFINE_PER_CPU(unsigned long, qstats[qstat_num]);
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
static DEFINE_PER_CPU(u64, pv_kick_time);
+#endif
/*
* Function to read and return the qlock statistical counter values
@@ -214,8 +256,8 @@ static int __init init_qspinlock_stat(void)
* performance.
*/
for (i = 0; i < qstat_num; i++)
- if (!debugfs_create_file(qstat_names[i], 0400, d_qstat,
- (void *)(long)i, &fops_qstat))
+ if (qstat_names[i] && !debugfs_create_file(qstat_names[i],
+ 0400, d_qstat, (void *)(long)i, &fops_qstat))
goto fail_undo;
if (!debugfs_create_file(qstat_names[qstat_reset_cnts], 0200, d_qstat,
@@ -232,7 +274,7 @@ static int __init init_qspinlock_stat(void)
fs_initcall(init_qspinlock_stat);
/*
- * Increment the PV qspinlock statistical counters
+ * Increment the qspinlock statistical counter.
*/
static inline void qstat_inc(enum qlock_stats stat, bool cond)
{
@@ -241,6 +283,20 @@ static inline void qstat_inc(enum qlock_stats stat, bool
cond)
}
/*
+ * Increment either one of the qspinlock statistical counters depending
+ * on the given condition.
+ */
+static inline void qstat_inc_either(enum qlock_stats true_stat,
+ enum qlock_stats false_stat, bool cond)
+{
+ if (cond)
+ this_cpu_inc(qstats[true_stat]);
+ else
+ this_cpu_inc(qstats[false_stat]);
+}
+
+#ifdef CONFIG_PARAVIRT_SPINLOCKS
+/*
* PV hash hop count
*/
static inline void qstat_hop(int hopcnt)
@@ -279,9 +335,14 @@ static inline void __pv_wait(u8 *ptr, u8 val)
#define pv_kick(c) __pv_kick(c)
#define pv_wait(p, v) __pv_wait(p, v)
+#endif /* CONFIG_PARAVIRT_SPINLOCKS */
+
#else /* CONFIG_QUEUED_LOCK_STAT */
static inline void qstat_inc(enum qlock_stats stat, bool cond) { }
+static inline void qstat_inc_either(enum qlock_stats true_stat,
+ enum qlock_stats false_stat, bool cond) { }
static inline void qstat_hop(int hopcnt) { }
#endif /* CONFIG_QUEUED_LOCK_STAT */
+#endif /* __KERNEL_LOCKING_QSPINLOCK_STAT_H */
--
1.8.3.1
