Now that the pmonr state machine and pqr_common are in place, add pmonr_update_sched_rmid to find the adequate rmid to use. With it, complete the body of PMUs functions that start/stop, add/del events.
A pmonr in Unused state tries to allocate a free rmid the first time one of its monitored threads is scheduled in a CPU package (lazy allocation of rmids). If there is no available rmids in that package, the pmonr enters a Dep_Idle state (it borrows the sched_rmid from its Lowest Monitored Ancestor (lma) pmonr). When a event is stopped, no other event runs in that CPU and PQR msr uses the rmid of the monr_hrchy_root's pmonr for that CPU's package. Details in pmonr state machine's comments. Signed-off-by: David Carrillo-Cisneros <davi...@google.com> --- arch/x86/events/intel/cmt.c | 101 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c index ce5be74..9421a3e 100644 --- a/arch/x86/events/intel/cmt.c +++ b/arch/x86/events/intel/cmt.c @@ -650,6 +650,74 @@ static int monr_append_event(struct monr *monr, struct perf_event *event) return err; } +/** + * pmonr_update_sched_rmid() - Update sched_rmid for @pmonr in current package. + * + * Always finds valid rmids for non Off pmonr. Safe to call with IRQs disabled. + * A lock-free fast path reuses the rmid when the pmonr has been scheduled + * before in this package. Otherwise, tries to get a free rmid. On failure, + * enters Dep_Idle state and uses the rmid of its lender. There is always a + * pmonr to borrow from since monr_hrchy_root has all its pmonrs in Active + * state. + * Return: new pmonr_rmids for pmonr. + */ +static inline union pmonr_rmids pmonr_update_sched_rmid(struct pmonr *pmonr) +{ + struct pkg_data *pkgd = pmonr->pkgd; + union pmonr_rmids rmids; + u32 free_rmid; + + /* Use atomic_rmids to check state in a lock-free fastpath. */ + rmids.value = atomic64_read(&pmonr->atomic_rmids); + if (rmids.sched_rmid != INVALID_RMID) + return rmids; + + /* No need to obtain RMID if in Off state. */ + if (rmids.sched_rmid == rmids.read_rmid) + return rmids; + + /* + * Lock-free path failed. Now acquire lock and verify that state + * and atomic_rmids haven't changed. If still Unused, try to + * obtain a free RMID. + */ + raw_spin_lock(&pkgd->lock); + + /* With lock acquired it is ok to read pmonr::state. */ + if (pmonr->state != PMONR_UNUSED) { + /* Update rmids in case they changed before acquiring lock. */ + rmids.value = atomic64_read(&pmonr->atomic_rmids); + raw_spin_unlock(&pkgd->lock); + return rmids; + } + + free_rmid = find_first_bit(pkgd->free_rmids, CMT_MAX_NR_RMIDS); + if (free_rmid == CMT_MAX_NR_RMIDS) + pmonr_unused_to_dep_idle(pmonr); + else + pmonr_unused_to_active(pmonr, free_rmid); + + raw_spin_unlock(&pkgd->lock); + + rmids.value = atomic64_read(&pmonr->atomic_rmids); + + return rmids; +} + +static inline union pmonr_rmids monr_get_sched_in_rmids(struct monr *monr) +{ + struct pmonr *pmonr; + union pmonr_rmids rmids; + u16 pkgid = topology_logical_package_id(smp_processor_id()); + + rcu_read_lock(); + pmonr = rcu_dereference(monr->pmonrs[pkgid]); + rmids = pmonr_update_sched_rmid(pmonr); + rcu_read_unlock(); + + return rmids; +} + static void monr_hrchy_insert_leaf(struct monr *monr, struct monr *parent) { unsigned long flags; @@ -865,16 +933,49 @@ static void intel_cmt_event_read(struct perf_event *event) { } +static inline void __intel_cmt_event_start(struct perf_event *event, + union pmonr_rmids rmids) +{ + if (!(event->hw.state & PERF_HES_STOPPED)) + return; + event->hw.state &= ~PERF_HES_STOPPED; + pqr_cache_update_rmid(rmids.sched_rmid); +} + static void intel_cmt_event_start(struct perf_event *event, int mode) { + union pmonr_rmids rmids; + + rmids = monr_get_sched_in_rmids(monr_from_event(event)); + __intel_cmt_event_start(event, rmids); } static void intel_cmt_event_stop(struct perf_event *event, int mode) { + union pmonr_rmids rmids; + + if (event->hw.state & PERF_HES_STOPPED) + return; + event->hw.state |= PERF_HES_STOPPED; + rmids = monr_get_sched_in_rmids(monr_hrchy_root); + /* + * HW tracks the rmid even when event is not scheduled and event + * reads occur even if event is Inactive. Therefore there is no need to + * read when event is stopped. + */ + pqr_cache_update_rmid(rmids.sched_rmid); } static int intel_cmt_event_add(struct perf_event *event, int mode) { + union pmonr_rmids rmids; + + event->hw.state = PERF_HES_STOPPED; + rmids = monr_get_sched_in_rmids(monr_from_event(event)); + + if (mode & PERF_EF_START) + __intel_cmt_event_start(event, rmids); + return 0; } -- 2.8.0.rc3.226.g39d4020