Introduce struct pkg_data to store per CPU package locks and data for
new CMT driver.
Each pkg_data is initialiazed/terminated on demand when the first/last CPU
in its package goes online/offline.
More details in code's comments.
Signed-off-by: David Carrillo-Cisneros <davi...@google.com>
---
arch/x86/events/intel/Makefile | 1 +
arch/x86/events/intel/cmt.c | 268 +++++++++++++++++++++++++++++++++++++++++
arch/x86/events/intel/cmt.h | 29 +++++
include/linux/cpuhotplug.h | 2 +
4 files changed, 300 insertions(+)
create mode 100644 arch/x86/events/intel/cmt.c
create mode 100644 arch/x86/events/intel/cmt.h
diff --git a/arch/x86/events/intel/Makefile b/arch/x86/events/intel/Makefile
index e9d8520..02fecbc 100644
--- a/arch/x86/events/intel/Makefile
+++ b/arch/x86/events/intel/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_PERF_EVENTS_INTEL_UNCORE) += intel-uncore.o
intel-uncore-objs := uncore.o uncore_nhmex.o uncore_snb.o
uncore_snbep.o
obj-$(CONFIG_PERF_EVENTS_INTEL_CSTATE) += intel-cstate.o
intel-cstate-objs := cstate.o
+obj-$(CONFIG_INTEL_RDT_M) += cmt.o
diff --git a/arch/x86/events/intel/cmt.c b/arch/x86/events/intel/cmt.c
new file mode 100644
index 0000000..267a9ec
--- /dev/null
+++ b/arch/x86/events/intel/cmt.c
@@ -0,0 +1,268 @@
+/*
+ * Intel Cache Monitoring Technology (CMT) support.
+ */
+
+#include <linux/slab.h>
+#include <asm/cpu_device_id.h>
+#include "cmt.h"
+#include "../perf_event.h"
+
+static DEFINE_MUTEX(cmt_mutex);
+
+static unsigned int cmt_l3_scale; /* cmt hw units to bytes. */
+
+static unsigned int __min_max_rmid; /* minimum max_rmid across all pkgs. */
+
+/* Array of packages (array of pkgds). It's protected by RCU or cmt_mutex. */
+static struct pkg_data **cmt_pkgs_data;
+
+/*
+ * If @pkgd == NULL, return first online, pkg_data in cmt_pkgs_data.
+ * Otherwise next online pkg_data or NULL if no more.
+ */
+static struct pkg_data *cmt_pkgs_data_next_rcu(struct pkg_data *pkgd)
+{
+ u16 p, nr_pkgs = topology_max_packages();
+
+ if (!pkgd)
+ return rcu_dereference_check(cmt_pkgs_data[0],
+ lockdep_is_held(&cmt_mutex));
+ p = pkgd->pkgid + 1;
+ pkgd = NULL;
+
+ while (!pkgd && p < nr_pkgs) {
+ pkgd = rcu_dereference_check(cmt_pkgs_data[p++],
+ lockdep_is_held(&cmt_mutex));
+ }
+
+ return pkgd;
+}
+
+static void free_pkg_data(struct pkg_data *pkg_data)
+{
+ kfree(pkg_data);
+}
+
+/* Init pkg_data for @cpu 's package. */
+static struct pkg_data *alloc_pkg_data(int cpu)
+{
+ struct cpuinfo_x86 *c = &cpu_data(cpu);
+ struct pkg_data *pkgd;
+ int numa_node = cpu_to_node(cpu);
+ u16 pkgid = topology_logical_package_id(cpu);
+
+ if (c->x86_cache_occ_scale != cmt_l3_scale) {
+ /* 0 scale must have been converted to 1 automatically. */
+ if (c->x86_cache_occ_scale || cmt_l3_scale != 1) {
+ pr_err("Multiple LLC scale values, disabling CMT
support.\n");
+ return ERR_PTR(-ENXIO);
+ }
+ }
+
+ pkgd = kzalloc_node(sizeof(*pkgd), GFP_KERNEL, numa_node);
+ if (!pkgd)
+ return ERR_PTR(-ENOMEM);
+
+ pkgd->max_rmid = c->x86_cache_max_rmid;
+
+ pkgd->work_cpu = cpu;
+ pkgd->pkgid = pkgid;
+
+ __min_max_rmid = min(__min_max_rmid, pkgd->max_rmid);
+
+ return pkgd;
+}
+
+static void __terminate_pkg_data(struct pkg_data *pkgd)
+{
+ lockdep_assert_held(&cmt_mutex);
+
+ free_pkg_data(pkgd);
+}
+
+static int init_pkg_data(int cpu)
+{
+ struct pkg_data *pkgd;
+ u16 pkgid = topology_logical_package_id(cpu);
+
+ lockdep_assert_held(&cmt_mutex);
+
+ /* Verify that this pkgid isn't already initialized. */
+ if (WARN_ON_ONCE(cmt_pkgs_data[pkgid]))
+ return -EPERM;
+
+ pkgd = alloc_pkg_data(cpu);
+ if (IS_ERR(pkgd))
+ return PTR_ERR(pkgd);
+
+ rcu_assign_pointer(cmt_pkgs_data[pkgid], pkgd);
+ synchronize_rcu();
+
+ return 0;
+}
+
+static int intel_cmt_hp_online_enter(unsigned int cpu)
+{
+ struct pkg_data *pkgd;
+ u16 pkgid = topology_logical_package_id(cpu);
+
+ rcu_read_lock();
+ pkgd = rcu_dereference(cmt_pkgs_data[pkgid]);
+ if (pkgd->work_cpu >= nr_cpu_ids)
+ pkgd->work_cpu = cpu;
+
+ rcu_read_unlock();
+
+ return 0;
+}
+
+static int intel_cmt_hp_online_exit(unsigned int cpu)
+{
+ struct pkg_data *pkgd;
+ u16 pkgid = topology_logical_package_id(cpu);
+
+ rcu_read_lock();
+ pkgd = rcu_dereference(cmt_pkgs_data[pkgid]);
+ if (pkgd->work_cpu == cpu)
+ pkgd->work_cpu = cpumask_any_but(
+ topology_core_cpumask(cpu), cpu);
+ rcu_read_unlock();
+
+ return 0;
+}
+
+static int intel_cmt_prep_up(unsigned int cpu)
+{
+ struct pkg_data *pkgd;
+ int err = 0;
+ u16 pkgid = topology_logical_package_id(cpu);
+
+ mutex_lock(&cmt_mutex);
+ pkgd = rcu_dereference_protected(cmt_pkgs_data[pkgid],
+ lockdep_is_held(&cmt_mutex));
+ if (!pkgd)
+ err = init_pkg_data(cpu);
+ mutex_unlock(&cmt_mutex);
+
+ return err;
+}
+
+static int intel_cmt_prep_down(unsigned int cpu)
+{
+ struct pkg_data *pkgd;
+ u16 pkgid = topology_logical_package_id(cpu);
+
+ mutex_lock(&cmt_mutex);
+ pkgd = rcu_dereference_protected(cmt_pkgs_data[pkgid],
+ lockdep_is_held(&cmt_mutex));
+ if (pkgd->work_cpu >= nr_cpu_ids) {
+ /* will destroy pkgd */
+ __terminate_pkg_data(pkgd);
+ RCU_INIT_POINTER(cmt_pkgs_data[pkgid], NULL);
+ synchronize_rcu();
+ }
+ mutex_unlock(&cmt_mutex);
+
+ return 0;
+}
+
+static const struct x86_cpu_id intel_cmt_match[] = {
+ { .vendor = X86_VENDOR_INTEL, .feature = X86_FEATURE_CMT_OCCUP_LLC },
+ {}
+};
+
+static void cmt_dealloc(void)
+{
+ kfree(cmt_pkgs_data);
+ cmt_pkgs_data = NULL;
+}
+
+static int __init cmt_alloc(void)
+{
+ cmt_l3_scale = boot_cpu_data.x86_cache_occ_scale;
+ if (cmt_l3_scale == 0)
+ cmt_l3_scale = 1;
+
+ cmt_pkgs_data = kcalloc(topology_max_packages(),
+ sizeof(*cmt_pkgs_data), GFP_KERNEL);
+ if (!cmt_pkgs_data)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static int __init cmt_start(void)
+{
+ char *str, scale[20];
+ int err;
+
+ /* will be modified by init_pkg_data() in intel_cmt_prep_up(). */
+ __min_max_rmid = UINT_MAX;
+ err = cpuhp_setup_state(CPUHP_PERF_X86_CMT_PREP,
+ "PERF_X86_CMT_PREP",
+ intel_cmt_prep_up,
+ intel_cmt_prep_down);
+ if (err)
+ return err;
+
+ err = cpuhp_setup_state(CPUHP_AP_PERF_X86_CMT_ONLINE,
+ "AP_PERF_X86_CMT_ONLINE",
+ intel_cmt_hp_online_enter,
+ intel_cmt_hp_online_exit);
+ if (err)
+ goto rm_prep;
+
+ snprintf(scale, sizeof(scale), "%u", cmt_l3_scale);
+ str = kstrdup(scale, GFP_KERNEL);
+ if (!str) {
+ err = -ENOMEM;
+ goto rm_online;
+ }
+
+ return 0;
+
+rm_online:
+ cpuhp_remove_state(CPUHP_AP_PERF_X86_CMT_ONLINE);
+rm_prep:
+ cpuhp_remove_state(CPUHP_PERF_X86_CMT_PREP);
+
+ return err;
+}
+
+static int __init intel_cmt_init(void)
+{
+ struct pkg_data *pkgd = NULL;
+ int err = 0;
+
+ if (!x86_match_cpu(intel_cmt_match)) {
+ err = -ENODEV;
+ goto err_exit;
+ }
+
+ err = cmt_alloc();
+ if (err)
+ goto err_exit;
+
+ err = cmt_start();
+ if (err)
+ goto err_dealloc;
+
+ pr_info("Intel CMT enabled with ");
+ rcu_read_lock();
+ while ((pkgd = cmt_pkgs_data_next_rcu(pkgd))) {
+ pr_cont("%d RMIDs for pkg %d, ",
+ pkgd->max_rmid + 1, pkgd->pkgid);
+ }
+ rcu_read_unlock();
+ pr_cont("and l3 scale of %d KBs.\n", cmt_l3_scale);
+
+ return err;
+
+err_dealloc:
+ cmt_dealloc();
+err_exit:
+ pr_err("Intel CMT registration failed with error: %d\n", err);
+ return err;
+}
+
+device_initcall(intel_cmt_init);
diff --git a/arch/x86/events/intel/cmt.h b/arch/x86/events/intel/cmt.h
new file mode 100644
index 0000000..8c16797
--- /dev/null
+++ b/arch/x86/events/intel/cmt.h
@@ -0,0 +1,29 @@
+/*
+ * Intel Monitoring Technology (CMT) support.
+ * (formerly Intel Cache QoS Monitoring, CQM)
+ *
+ *
+ * Locking
+ *
+ * One global cmt_mutex. One mutex and spin_lock per package.
+ * cmt_pkgs_data is RCU protected.
+ *
+ * Rules:
+ * - cmt_mutex: Hold for CMT init/terminate, event init/terminate,
+ * cgroup start/stop.
+ */
+
+/**
+ * struct pkg_data - Per-package CMT data.
+ *
+ * @work_cpu: CPU to run rotation and other batch jobs.
+ * It must be in the package associated to its
+ * instance of pkg_data.
+ * @max_rmid: Max rmid valid for CPUs in this package.
+ * @pkgid: The logical package id for this pkgd.
+ */
+struct pkg_data {
+ unsigned int work_cpu;
+ u32 max_rmid;
+ u16 pkgid;
+};
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 320a3be..604660a 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -11,6 +11,7 @@ enum cpuhp_state {
CPUHP_PERF_X86_UNCORE_PREP,
CPUHP_PERF_X86_AMD_UNCORE_PREP,
CPUHP_PERF_X86_RAPL_PREP,
+ CPUHP_PERF_X86_CMT_PREP,
CPUHP_PERF_BFIN,
CPUHP_PERF_POWER,
CPUHP_PERF_SUPERH,
@@ -110,6 +111,7 @@ enum cpuhp_state {
CPUHP_AP_PERF_X86_AMD_UNCORE_ONLINE,
CPUHP_AP_PERF_X86_AMD_POWER_ONLINE,
CPUHP_AP_PERF_X86_RAPL_ONLINE,
+ CPUHP_AP_PERF_X86_CMT_ONLINE,
CPUHP_AP_PERF_X86_CSTATE_ONLINE,
CPUHP_AP_PERF_S390_CF_ONLINE,
CPUHP_AP_PERF_S390_SF_ONLINE,
--
2.8.0.rc3.226.g39d4020
