Add support to manage CLOSid(CLass Of Service id) and capacity
bitmask(cbm) for code data prioritization(CDP).
Closid management includes changes to allocating, freeing closid and
closid_get and closid_put and changes to closid availability map during
mode switch. CDP has a separate cbm for code and data.
Once the mode is switched to cdp, the number of CLOSids is halved.
The clos_cbm_table is reused to store dcache_cbm and
icache_cbm entries and the index is calculated as below:
index of dcache_cbm for a CLOSid 'n' = (n << 1)
index of icache_cbm for a CLOSid 'n' = (n << 1) + 1.
The offset of the IA32_L3_MASK_n MSRs from the base is related to the
CLOSid 'n' in the same way.
In other words, each closid is mapped to a (dcache_cbm, icache_cbm) pair
when cdp mode is enabled. Support for setting up of the
clos_cbm_table and closmap when the mode switch happens is added.
Signed-off-by: Vikas Shivappa <vikas.shiva...@linux.intel.com>
Signed-off-by: Fenghua Yu <fenghua...@intel.com>
---
arch/x86/kernel/cpu/intel_rdt.c | 194 +++++++++++++++++++++++++++++++++++++++-
1 file changed, 190 insertions(+), 4 deletions(-)
diff --git a/arch/x86/kernel/cpu/intel_rdt.c b/arch/x86/kernel/cpu/intel_rdt.c
index 54a8e29..eab9d73 100644
--- a/arch/x86/kernel/cpu/intel_rdt.c
+++ b/arch/x86/kernel/cpu/intel_rdt.c
@@ -30,7 +30,13 @@
#include <asm/intel_rdt.h>
/*
- * cctable maintains 1:1 mapping between CLOSid and cache bitmask.
+ * During cache alloc mode cctable maintains 1:1 mapping between
+ * CLOSid and l3_cbm.
+ *
+ * During CDP mode, the cctable maintains a 1:2 mapping between the closid
+ * and (dcache_cbm, icache_cbm) pair.
+ * index of a dcache_cbm for CLOSid 'n' = n << 1.
+ * index of a icache_cbm for CLOSid 'n' = n << 1 + 1
*/
static struct clos_cbm_table *cctable;
/*
@@ -53,6 +59,13 @@ static bool cdp_enabled;
#define __DCBM_TABLE_INDEX(x) (x << 1)
#define __ICBM_TABLE_INDEX(x) ((x << 1) + 1)
+#define __DCBM_MSR_INDEX(x) \
+ CBM_FROM_INDEX(__DCBM_TABLE_INDEX(x))
+#define __ICBM_MSR_INDEX(x) \
+ CBM_FROM_INDEX(__ICBM_TABLE_INDEX(x))
+
+#define DCBM_TABLE_INDEX(x) (x << cdp_enabled)
+#define ICBM_TABLE_INDEX(x) ((x << cdp_enabled) + cdp_enabled)
static struct intel_rdt rdt_root_group;
#define rdt_for_each_child(pos_css, parent_ir) \
@@ -133,9 +146,12 @@ static inline void closid_tasks_sync(void)
on_each_cpu_mask(cpu_online_mask, __intel_rdt_sched_in, NULL, 1);
}
+/*
+ * When cdp mode is enabled, refcnt is maintained in the dcache_cbm entry.
+ */
static inline void closid_get(u32 closid)
{
- struct clos_cbm_table *cct = &cctable[closid];
+ struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
lockdep_assert_held(&rdt_group_mutex);
@@ -165,7 +181,7 @@ static int closid_alloc(u32 *closid)
static inline void closid_free(u32 closid)
{
clear_bit(closid, cconfig.closmap);
- cctable[closid].l3_cbm = 0;
+ cctable[DCBM_TABLE_INDEX(closid)].l3_cbm = 0;
if (WARN_ON(!cconfig.closids_used))
return;
@@ -175,7 +191,7 @@ static inline void closid_free(u32 closid)
static void closid_put(u32 closid)
{
- struct clos_cbm_table *cct = &cctable[closid];
+ struct clos_cbm_table *cct = &cctable[DCBM_TABLE_INDEX(closid)];
lockdep_assert_held(&rdt_group_mutex);
if (WARN_ON(!cct->clos_refcnt))
@@ -259,6 +275,30 @@ static bool cbm_search(unsigned long cbm, u32 *closid)
return false;
}
+static bool cbm_pair_search(unsigned long dcache_cbm, unsigned long icache_cbm,
+ u32 *closid)
+{
+ u32 maxid = cconfig.max_closid;
+ unsigned long dcbm, icbm;
+ u32 i, dindex, iindex;
+
+ for (i = 0; i < maxid; i++) {
+ dindex = __DCBM_TABLE_INDEX(i);
+ iindex = __ICBM_TABLE_INDEX(i);
+ dcbm = cctable[dindex].l3_cbm;
+ icbm = cctable[iindex].l3_cbm;
+
+ if (cctable[dindex].clos_refcnt &&
+ bitmap_equal(&dcache_cbm, &dcbm, MAX_CBM_LENGTH) &&
+ bitmap_equal(&icache_cbm, &icbm, MAX_CBM_LENGTH)) {
+ *closid = i;
+ return true;
+ }
+ }
+
+ return false;
+}
+
static void closcbm_map_dump(void)
{
u32 i;
@@ -289,6 +329,93 @@ static inline void msr_update_all(int msr, u64 val)
on_each_cpu_mask(&rdt_cpumask, msr_cpu_update, &info, 1);
}
+/*
+ * clos_cbm_table_df() - Defragments the clos_cbm_table entries
+ * @ct: The clos_cbm_table to which the defragmented entries are copied.
+ *
+ * The max entries in ct is never > original max closids / 2.
+ */
+static void clos_cbm_table_df(struct clos_cbm_table *ct)
+{
+ u32 orig_maxid = boot_cpu_data.x86_cache_max_closid;
+ int i, j;
+
+ for (i = 0, j = 0; i < orig_maxid; i++) {
+ if (cctable[i].clos_refcnt) {
+ ct[j] = cctable[i];
+ set_bit(j, cconfig.closmap);
+ j++;
+ }
+ }
+}
+
+/*
+ * post_cdp_enable() - This sets up the clos_cbm_table and
+ * IA32_L3_MASK_n MSRs before starting to use CDP.
+ *
+ * The existing l3_cbm entries are retained as dcache_cbm and
+ * icache_cbm entries. The IA32_L3_QOS_n MSRs are also updated
+ * as they were reset to all 1s before mode change.
+ */
+static int post_cdp_enable(void)
+{
+ u32 orig_maxid = boot_cpu_data.x86_cache_max_closid;
+ u32 maxid = cconfig.max_closid;
+ int size, dindex, iindex, i;
+ struct clos_cbm_table *ct;
+
+ maxid = cconfig.max_closid;
+ size = maxid * sizeof(struct clos_cbm_table);
+ ct = kzalloc(size, GFP_KERNEL);
+ if (!ct)
+ return -ENOMEM;
+
+ bitmap_zero(cconfig.closmap, orig_maxid);
+ clos_cbm_table_df(ct);
+
+ for (i = 0; i < maxid; i++) {
+ if (ct[i].clos_refcnt) {
+ msr_update_all(__DCBM_MSR_INDEX(i), ct[i].l3_cbm);
+ msr_update_all(__ICBM_MSR_INDEX(i), ct[i].l3_cbm);
+ }
+ dindex = __DCBM_TABLE_INDEX(i);
+ iindex = __ICBM_TABLE_INDEX(i);
+ cctable[dindex] = cctable[iindex] = ct[i];
+ }
+ kfree(ct);
+
+ return 0;
+}
+
+/*
+ * post_cdp_disable() - Set the state of closmap and clos_cbm_table
+ * before using the cache alloc mode.
+ *
+ * The existing dcache_cbm entries are retained as l3_cbm entries.
+ * The IA32_L3_QOS_n MSRs are also updated
+ * as they were reset to all 1s before mode change.
+ */
+static void post_cdp_disable(void)
+{
+ int dindex, maxid, i;
+
+ maxid = cconfig.max_closid >> 1;
+ for (i = 0; i < maxid; i++) {
+ dindex = __DCBM_TABLE_INDEX(i);
+ if (cctable[dindex].clos_refcnt)
+ msr_update_all(CBM_FROM_INDEX(i),
+ cctable[dindex].l3_cbm);
+
+ cctable[i] = cctable[dindex];
+ }
+
+ /*
+ * We updated half of the clos_cbm_table entries, initialize the
+ * rest of the clos_cbm_table entries.
+ */
+ memset(&cctable[maxid], 0, maxid * sizeof(struct clos_cbm_table));
+}
+
static bool code_data_mask_equal(void)
{
int i, dindex, iindex;
@@ -304,6 +431,65 @@ static bool code_data_mask_equal(void)
return true;
}
+static void __switch_mode_cdp(bool cdpenable)
+{
+ u32 max_cbm_len = boot_cpu_data.x86_cache_max_cbm_len;
+ u32 orig_maxid = boot_cpu_data.x86_cache_max_closid;
+ u32 max_cbm, i;
+
+ max_cbm = (1ULL << max_cbm_len) - 1;
+ for (i = 0; i < orig_maxid; i++)
+ msr_update_all(CBM_FROM_INDEX(i), max_cbm);
+
+ msr_update_all(MSR_IA32_PQOS_CFG, cdpenable);
+
+ if (cdpenable)
+ cconfig.max_closid = cconfig.max_closid >> 1;
+ else
+ cconfig.max_closid = cconfig.max_closid << 1;
+ cdp_enabled = cdpenable;
+}
+
+/*
+ * switch_mode_cdp() - switch between legacy cache alloc and cdp modes
+ * @cdpmode: '0' to disable cdp and '1' to enable cdp
+ *
+ * cdp is enabled only when the number of closids used is less than half
+ * of available closids. Switch to legacy cache alloc mode when
+ * for each (dcache_cbm,icache_cbm) pair, the dcache_cbm = icache_cbm.
+ */
+static int switch_mode_cdp(bool cdpenable)
+{
+ struct cpuinfo_x86 *c = &boot_cpu_data;
+ u32 maxid = cconfig.max_closid;
+ int err = 0;
+
+ lockdep_assert_held(&rdt_group_mutex);
+
+ if (!cpu_has(c, X86_FEATURE_CDP_L3) || cdpenable == cdp_enabled)
+ return -EINVAL;
+
+ if ((cdpenable && (cconfig.closids_used >= (maxid >> 1))) ||
+ (!cdpenable && !code_data_mask_equal()))
+ return -ENOSPC;
+
+ __switch_mode_cdp(cdpenable);
+
+ /*
+ * After mode switch to cdp, the index for IA32_L3_MASK_n from the base
+ * for a CLOSid 'n' is:
+ * dcache_cbm_index (n) = (n << 1)
+ * icache_cbm_index (n) = (n << 1) +1
+ */
+ if (cdpenable)
+ err = post_cdp_enable();
+ else
+ post_cdp_disable();
+ closcbm_map_dump();
+
+ return err;
+}
+
static inline bool rdt_cpumask_update(int cpu)
{
cpumask_and(&tmp_cpumask, &rdt_cpumask, topology_core_cpumask(cpu));
--
1.8.1.2
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
