For perf_events, PQR_ASSOC is written twice: in ctx_in and ctx_out.
Since writes to PQR_ASSOC are slow, this has a high impact in context
swich cost.
Modify pqr_common so pqr_update_rmid do not write the msr
right away, but until a call to __pqr_ctx_switch that is called from
the recently introduced finish_arch_pre_lock_switch hook.
Reviewed-by: Stephane Eranian <eran...@google.com>
Signed-off-by: David Carrillo-Cisneros <davi...@google.com>
---
arch/x86/events/intel/cqm.c | 12 +++++++-----
arch/x86/include/asm/pqr_common.h | 21 ++++++++++++++-------
arch/x86/kernel/cpu/pqr_common.c | 11 ++++++++++-
3 files changed, 31 insertions(+), 13 deletions(-)
diff --git a/arch/x86/events/intel/cqm.c b/arch/x86/events/intel/cqm.c
index 5928bdb..83b041a 100644
--- a/arch/x86/events/intel/cqm.c
+++ b/arch/x86/events/intel/cqm.c
@@ -2515,7 +2515,7 @@ static inline void __intel_cqm_event_start(
if (!(event->hw.state & PERF_HES_STOPPED))
return;
event->hw.state &= ~PERF_HES_STOPPED;
- pqr_update_rmid(summary.sched_rmid, PQR_RMID_MODE_EVENT);
+ pqr_cache_update_rmid(summary.sched_rmid, PQR_RMID_MODE_EVENT);
}
static void intel_cqm_event_start(struct perf_event *event, int mode)
@@ -2545,7 +2545,7 @@ static void intel_cqm_event_stop(struct perf_event
*event, int mode)
/* Occupancy of CQM events is obtained at read. No need to read
* when event is stopped since read on inactive cpus succeed.
*/
- pqr_update_rmid(summary.sched_rmid, PQR_RMID_MODE_NOEVENT);
+ pqr_cache_update_rmid(summary.sched_rmid, PQR_RMID_MODE_NOEVENT);
}
static int intel_cqm_event_add(struct perf_event *event, int mode)
@@ -2962,8 +2962,10 @@ static void intel_cqm_cpu_starting(unsigned int cpu)
u16 pkg_id = topology_physical_package_id(cpu);
state->rmid = 0;
- state->rmid_mode = PQR_RMID_MODE_NOEVENT;
+ state->next_rmid = 0;
+ state->next_rmid_mode = PQR_RMID_MODE_NOEVENT;
state->closid = 0;
+ state->next_closid = 0;
/* XXX: lock */
/* XXX: Make sure this case is handled when hotplug happens. */
@@ -3190,12 +3192,12 @@ inline void __intel_cqm_no_event_sched_in(void)
if (WARN_ON_ONCE(!__valid_rmid(pkg_id, summary.sched_rmid)))
goto no_rmid;
- pqr_update_rmid(summary.sched_rmid, PQR_RMID_MODE_NOEVENT);
+ pqr_cache_update_rmid(summary.sched_rmid, PQR_RMID_MODE_NOEVENT);
return;
no_rmid:
summary.value = atomic64_read(&root_pmonr->prmid_summary_atomic);
- pqr_update_rmid(summary.sched_rmid, PQR_RMID_MODE_NOEVENT);
+ pqr_cache_update_rmid(summary.sched_rmid, PQR_RMID_MODE_NOEVENT);
#endif
}
diff --git a/arch/x86/include/asm/pqr_common.h
b/arch/x86/include/asm/pqr_common.h
index 0af04d2..bfaa7d8 100644
--- a/arch/x86/include/asm/pqr_common.h
+++ b/arch/x86/include/asm/pqr_common.h
@@ -26,8 +26,10 @@ enum intel_pqr_rmid_mode {
/**
* struct intel_pqr_state - State cache for the PQR MSR
* @rmid: Last RMID written to hw.
+ * @next_rmid: Next RMID to write to hw.
* @rmid_mode: Next RMID's mode.
* @closid: The current Class Of Service ID
+ * @next_closid: The Class Of Service ID to use.
*
* The upper 32 bits of MSR_IA32_PQR_ASSOC contain closid and the
* lower 10 bits rmid. The update to MSR_IA32_PQR_ASSOC always
@@ -39,22 +41,27 @@ enum intel_pqr_rmid_mode {
*/
struct intel_pqr_state {
u32 rmid;
- enum intel_pqr_rmid_mode rmid_mode;
+ u32 next_rmid;
+ enum intel_pqr_rmid_mode next_rmid_mode;
u32 closid;
+ u32 next_closid;
};
DECLARE_PER_CPU(struct intel_pqr_state, pqr_state);
-static inline void pqr_update_rmid(u32 rmid, enum intel_pqr_rmid_mode mode)
+static inline void pqr_cache_update_rmid(u32 rmid, enum intel_pqr_rmid_mode
mode)
{
struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
- state->rmid_mode = mode;
+ state->next_rmid_mode = mode;
+ state->next_rmid = rmid;
+}
+
+static inline void pqr_cache_update_closid(u32 closid)
+{
+ struct intel_pqr_state *state = this_cpu_ptr(&pqr_state);
- if (state->rmid == rmid)
- return;
- state->rmid = rmid;
- wrmsr(MSR_IA32_PQR_ASSOC, rmid, state->closid);
+ state->next_closid = closid;
}
void __pqr_ctx_switch(void);
diff --git a/arch/x86/kernel/cpu/pqr_common.c b/arch/x86/kernel/cpu/pqr_common.c
index e36702f..0ae2b2c 100644
--- a/arch/x86/kernel/cpu/pqr_common.c
+++ b/arch/x86/kernel/cpu/pqr_common.c
@@ -19,6 +19,15 @@ inline void __pqr_ctx_switch(void)
/* If perf_event did set rmid that is used, do not try
* to obtain another one from current task.
*/
- if (state->rmid_mode == PQR_RMID_MODE_NOEVENT)
+ if (state->next_rmid_mode == PQR_RMID_MODE_NOEVENT)
__intel_cqm_no_event_sched_in();
+
+ /* __intel_cqm_no_event_sched_in might have changed next_rmid. */
+ if (state->rmid == state->next_rmid &&
+ state->closid == state->next_closid)
+ return;
+
+ state->rmid = state->next_rmid;
+ state->closid = state->next_closid;
+ wrmsr(MSR_IA32_PQR_ASSOC, state->rmid, state->closid);
}
--
2.8.0.rc3.226.g39d4020
