[Xen-devel] [PATCH] tools/misc: add AVX512 vpopcntdq in xen-cpuid.c

[He Chen]

Add AVX512 vpopcntdq information in xen-cpuid.c

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 tools/misc/xen-cpuid.c | 6 +-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tools/misc/xen-cpuid.c b/tools/misc/xen-cpuid.c
index 5d66e94..106be0f 100644
--- a/tools/misc/xen-cpuid.c
+++ b/tools/misc/xen-cpuid.c
@@ -124,7 +124,11 @@ static const char *str_7c0[32] =
 [ 2] = "REZ",  [ 3] = "pku",
 [ 4] = "ospke",
 
-[5 ... 31] = "REZ",
+[5 ... 13] = "REZ",
+
+[14] = "avx512_vpopcntdq",
+
+[15 ... 31] = "REZ",
 };
 
 static const char *str_e7d[32] =
-- 
2.7.4


___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH v2] x86/cpuid: Add AVX512_VPOPCNTDQ support

[He Chen]

AVX512_VPOPCNTDQ: Vector POPCNT instructions for word and qwords.
variable precision.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes from v1:
renanme VPOPCNTDQ to AVX512_VPOPCNTDQ.
---
 xen/include/public/arch-x86/cpufeatureset.h | 1 +
 xen/tools/gen-cpuid.py  | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/xen/include/public/arch-x86/cpufeatureset.h 
b/xen/include/public/arch-x86/cpufeatureset.h
index 565ccd5..c9b38e5 100644
--- a/xen/include/public/arch-x86/cpufeatureset.h
+++ b/xen/include/public/arch-x86/cpufeatureset.h
@@ -226,6 +226,7 @@ XEN_CPUFEATURE(PREFETCHWT1,   6*32+ 0) /*A  PREFETCHWT1 
instruction */
 XEN_CPUFEATURE(AVX512VBMI,6*32+ 1) /*A  AVX-512 Vector Byte Manipulation 
Instrs */
 XEN_CPUFEATURE(PKU,   6*32+ 3) /*H  Protection Keys for Userspace */
 XEN_CPUFEATURE(OSPKE, 6*32+ 4) /*!  OS Protection Keys Enable */
+XEN_CPUFEATURE(AVX512_VPOPCNTDQ, 6*32+14) /*A  POPCNT for vectors of DW/QW */
 
 /* AMD-defined CPU features, CPUID level 0x8007.edx, word 7 */
 XEN_CPUFEATURE(ITSC,  7*32+ 8) /*   Invariant TSC */
diff --git a/xen/tools/gen-cpuid.py b/xen/tools/gen-cpuid.py
index 420a5cc..4a673b0 100755
--- a/xen/tools/gen-cpuid.py
+++ b/xen/tools/gen-cpuid.py
@@ -255,7 +255,8 @@ def crunch_numbers(state):
 # 512bit registers, and the instructions themselves. All further 
AVX512 features
 # are built on top of AVX512F
 AVX512F: [AVX512DQ, AVX512IFMA, AVX512PF, AVX512ER, AVX512CD,
-  AVX512BW, AVX512VL, AVX512VBMI, AVX512_4VNNIW, 
AVX512_4FMAPS],
+  AVX512BW, AVX512VL, AVX512VBMI, AVX512_4VNNIW,
+  AVX512_4FMAPS, AVX512_VPOPCNTDQ],
 }
 
 deep_features = tuple(sorted(deps.keys()))
-- 
2.7.4


___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH] x86/cpuid: Add AVX512_VPOPCNTDQ support

[He Chen]

On Tue, Jan 10, 2017 at 01:49:06AM -0700, Jan Beulich wrote:
> >>> On 10.01.17 at 07:34,  wrote:
> > --- a/xen/include/public/arch-x86/cpufeatureset.h
> > +++ b/xen/include/public/arch-x86/cpufeatureset.h
> > @@ -226,6 +226,7 @@ XEN_CPUFEATURE(PREFETCHWT1,   6*32+ 0) /*A  PREFETCHWT1 
> > instruction */
> >  XEN_CPUFEATURE(AVX512VBMI,6*32+ 1) /*A  AVX-512 Vector Byte 
> > Manipulation Instrs */
> >  XEN_CPUFEATURE(PKU,   6*32+ 3) /*H  Protection Keys for Userspace 
> > */
> >  XEN_CPUFEATURE(OSPKE, 6*32+ 4) /*!  OS Protection Keys Enable */
> > +XEN_CPUFEATURE(VPOPCNTDQ, 6*32+14) /*A  POPCNT for vectors of DW/QW */
> 
> Hmm, the document has an AVX512_ prefix here, and while that's a
> little lengthy I think we should try to stick to SDM names.
> 
Sure.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH] x86/cpuid: Add AVX512_VPOPCNTDQ support

[He Chen]

AVX512_VPOPCNTDQ: Vector POPCNT instructions for word and qwords.
variable precision.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/include/public/arch-x86/cpufeatureset.h | 1 +
 xen/tools/gen-cpuid.py  | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/xen/include/public/arch-x86/cpufeatureset.h 
b/xen/include/public/arch-x86/cpufeatureset.h
index 565ccd5..285224d 100644
--- a/xen/include/public/arch-x86/cpufeatureset.h
+++ b/xen/include/public/arch-x86/cpufeatureset.h
@@ -226,6 +226,7 @@ XEN_CPUFEATURE(PREFETCHWT1,   6*32+ 0) /*A  PREFETCHWT1 
instruction */
 XEN_CPUFEATURE(AVX512VBMI,6*32+ 1) /*A  AVX-512 Vector Byte Manipulation 
Instrs */
 XEN_CPUFEATURE(PKU,   6*32+ 3) /*H  Protection Keys for Userspace */
 XEN_CPUFEATURE(OSPKE, 6*32+ 4) /*!  OS Protection Keys Enable */
+XEN_CPUFEATURE(VPOPCNTDQ, 6*32+14) /*A  POPCNT for vectors of DW/QW */
 
 /* AMD-defined CPU features, CPUID level 0x8007.edx, word 7 */
 XEN_CPUFEATURE(ITSC,  7*32+ 8) /*   Invariant TSC */
diff --git a/xen/tools/gen-cpuid.py b/xen/tools/gen-cpuid.py
index 420a5cc..1067c85 100755
--- a/xen/tools/gen-cpuid.py
+++ b/xen/tools/gen-cpuid.py
@@ -255,7 +255,8 @@ def crunch_numbers(state):
 # 512bit registers, and the instructions themselves. All further 
AVX512 features
 # are built on top of AVX512F
 AVX512F: [AVX512DQ, AVX512IFMA, AVX512PF, AVX512ER, AVX512CD,
-  AVX512BW, AVX512VL, AVX512VBMI, AVX512_4VNNIW, 
AVX512_4FMAPS],
+  AVX512BW, AVX512VL, AVX512VBMI, AVX512_4VNNIW,
+  AVX512_4FMAPS, VPOPCNTDQ],
 }
 
 deep_features = tuple(sorted(deps.keys()))
-- 
2.7.4


___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH v2] x86/cpuid: Add AVX512_4VNNIW and AVX512_4FMAPS support

[He Chen]

Add two new AVX512 subfeatures support for guest.

AVX512_4VNNIW:
Vector instructions for deep learning enhanced word variable precision.

AVX512_4FMAPS:
Vector instructions for deep learning floating-point single precision.

Signed-off-by: Luwei Kang <luwei.k...@intel.com>
Signed-off-by: He Chen <he.c...@linux.intel.com>
---

Changes from v1:
Add new leaf in xen-cpuid.c

---
 tools/libxc/xc_cpuid_x86.c  |  8 ++--
 tools/misc/xen-cpuid.c  | 10 ++
 xen/arch/x86/cpu/common.c   |  2 +-
 xen/arch/x86/cpuid.c|  2 +-
 xen/arch/x86/hvm/hvm.c  |  1 +
 xen/arch/x86/traps.c|  5 +++--
 xen/include/asm-x86/cpuid.h |  1 +
 xen/include/public/arch-x86/cpufeatureset.h |  4 
 xen/tools/gen-cpuid.py  |  2 +-
 9 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/tools/libxc/xc_cpuid_x86.c b/tools/libxc/xc_cpuid_x86.c
index 2ad9aeb..e9e3691 100644
--- a/tools/libxc/xc_cpuid_x86.c
+++ b/tools/libxc/xc_cpuid_x86.c
@@ -547,13 +547,15 @@ static void xc_cpuid_hvm_policy(xc_interface *xch,
 {
 regs[1] = info->featureset[featureword_of(X86_FEATURE_FSGSBASE)];
 regs[2] = 
info->featureset[featureword_of(X86_FEATURE_PREFETCHWT1)];
+regs[3] = 
info->featureset[featureword_of(X86_FEATURE_AVX512_4VNNIW)];
 }
 else
 {
 regs[1] = 0;
 regs[2] = 0;
+regs[3] = 0;
 }
-regs[0] = regs[3] = 0;
+regs[0] = 0;
 break;
 
 case 0x000d:
@@ -638,13 +640,15 @@ static void xc_cpuid_pv_policy(xc_interface *xch,
 {
 regs[1] = info->featureset[featureword_of(X86_FEATURE_FSGSBASE)];
 regs[2] = 
info->featureset[featureword_of(X86_FEATURE_PREFETCHWT1)];
+regs[3] = 
info->featureset[featureword_of(X86_FEATURE_AVX512_4VNNIW)];
 }
 else
 {
 regs[1] = 0;
 regs[2] = 0;
+regs[3] = 0;
 }
-regs[0] = regs[3] = 0;
+regs[0] = 0;
 break;
 
 case 0x000d:
diff --git a/tools/misc/xen-cpuid.c b/tools/misc/xen-cpuid.c
index 44991f6..5d66e94 100644
--- a/tools/misc/xen-cpuid.c
+++ b/tools/misc/xen-cpuid.c
@@ -143,6 +143,15 @@ static const char *str_e8b[32] =
 [1 ... 31] = "REZ",
 };
 
+static const char *str_7d0[32] =
+{
+[0 ... 1] = "REZ",
+
+[ 2] = "avx512_4vnniw", [ 3] = "avx512_4fmaps",
+
+[4 ... 31] = "REZ",
+};
+
 static struct {
 const char *name;
 const char *abbr;
@@ -158,6 +167,7 @@ static struct {
 { "0x0007:0.ecx", "7c0", str_7c0 },
 { "0x8007.edx",   "e7d", str_e7d },
 { "0x8008.ebx",   "e8b", str_e8b },
+{ "0x0007:0.edx", "7d0", str_7d0 },
 };
 
 #define COL_ALIGN "18"
diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c
index 3475198..873 100644
--- a/xen/arch/x86/cpu/common.c
+++ b/xen/arch/x86/cpu/common.c
@@ -325,7 +325,7 @@ static void generic_identify(struct cpuinfo_x86 *c)
cpuid_count(0x0007, 0, ,

>x86_capability[cpufeat_word(X86_FEATURE_FSGSBASE)],
>x86_capability[cpufeat_word(X86_FEATURE_PKU)],
-   );
+   
>x86_capability[cpufeat_word(X86_FEATURE_AVX512_4VNNIW)]);
 }
 
 /*
diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c
index 63b2db9..3e85a63 100644
--- a/xen/arch/x86/cpuid.c
+++ b/xen/arch/x86/cpuid.c
@@ -78,7 +78,7 @@ static void __init calculate_raw_featureset(void)
 cpuid_count(0x7, 0, ,
 _featureset[FEATURESET_7b0],
 _featureset[FEATURESET_7c0],
-);
+_featureset[FEATURESET_7d0]);
 if ( max >= 0xd )
 cpuid_count(0xd, 1,
 _featureset[FEATURESET_Da1],
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 704fd64..752e5fb 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -3503,6 +3503,7 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
unsigned int *ebx,
  special_features[FEATURESET_7b0]);
 
 *ecx &= hvm_featureset[FEATURESET_7c0];
+*edx &= hvm_featureset[FEATURESET_7d0];
 
 /* Don't expose HAP-only features to non-hap guests. */
 if ( !hap_enabled(d) )
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index d56d76e..01ac1b1 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -1133,6 +1133,7 @@ void pv_cpuid(struct cpu_user_regs *regs)
   special_features[FEATURESET_7b0]);
 
 c &= pv_feat

[Xen-devel] [PATCH] x86/cpuid: Add AVX512_4VNNIW and AVX512_4FMAPS support

[He Chen]

Add two new AVX512 subfeatures support for guest.

AVX512_4VNNIW:
Vector instructions for deep learning enhanced word variable precision.

AVX512_4FMAPS:
Vector instructions for deep learning floating-point single precision.

Signed-off-by: Luwei Kang <luwei.k...@intel.com>
Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 tools/libxc/xc_cpuid_x86.c  | 8 ++--
 xen/arch/x86/cpu/common.c   | 2 +-
 xen/arch/x86/cpuid.c| 2 +-
 xen/arch/x86/hvm/hvm.c  | 1 +
 xen/arch/x86/traps.c| 5 +++--
 xen/include/asm-x86/cpuid.h | 1 +
 xen/include/public/arch-x86/cpufeatureset.h | 4 
 xen/tools/gen-cpuid.py  | 2 +-
 8 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/tools/libxc/xc_cpuid_x86.c b/tools/libxc/xc_cpuid_x86.c
index 2ad9aeb..e9e3691 100644
--- a/tools/libxc/xc_cpuid_x86.c
+++ b/tools/libxc/xc_cpuid_x86.c
@@ -547,13 +547,15 @@ static void xc_cpuid_hvm_policy(xc_interface *xch,
 {
 regs[1] = info->featureset[featureword_of(X86_FEATURE_FSGSBASE)];
 regs[2] = 
info->featureset[featureword_of(X86_FEATURE_PREFETCHWT1)];
+regs[3] = 
info->featureset[featureword_of(X86_FEATURE_AVX512_4VNNIW)];
 }
 else
 {
 regs[1] = 0;
 regs[2] = 0;
+regs[3] = 0;
 }
-regs[0] = regs[3] = 0;
+regs[0] = 0;
 break;
 
 case 0x000d:
@@ -638,13 +640,15 @@ static void xc_cpuid_pv_policy(xc_interface *xch,
 {
 regs[1] = info->featureset[featureword_of(X86_FEATURE_FSGSBASE)];
 regs[2] = 
info->featureset[featureword_of(X86_FEATURE_PREFETCHWT1)];
+regs[3] = 
info->featureset[featureword_of(X86_FEATURE_AVX512_4VNNIW)];
 }
 else
 {
 regs[1] = 0;
 regs[2] = 0;
+regs[3] = 0;
 }
-regs[0] = regs[3] = 0;
+regs[0] = 0;
 break;
 
 case 0x000d:
diff --git a/xen/arch/x86/cpu/common.c b/xen/arch/x86/cpu/common.c
index 3475198..873 100644
--- a/xen/arch/x86/cpu/common.c
+++ b/xen/arch/x86/cpu/common.c
@@ -325,7 +325,7 @@ static void generic_identify(struct cpuinfo_x86 *c)
cpuid_count(0x0007, 0, ,

>x86_capability[cpufeat_word(X86_FEATURE_FSGSBASE)],
>x86_capability[cpufeat_word(X86_FEATURE_PKU)],
-   );
+   
>x86_capability[cpufeat_word(X86_FEATURE_AVX512_4VNNIW)]);
 }
 
 /*
diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c
index 63b2db9..3e85a63 100644
--- a/xen/arch/x86/cpuid.c
+++ b/xen/arch/x86/cpuid.c
@@ -78,7 +78,7 @@ static void __init calculate_raw_featureset(void)
 cpuid_count(0x7, 0, ,
 _featureset[FEATURESET_7b0],
 _featureset[FEATURESET_7c0],
-);
+_featureset[FEATURESET_7d0]);
 if ( max >= 0xd )
 cpuid_count(0xd, 1,
 _featureset[FEATURESET_Da1],
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index 704fd64..752e5fb 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -3503,6 +3503,7 @@ void hvm_cpuid(unsigned int input, unsigned int *eax, 
unsigned int *ebx,
  special_features[FEATURESET_7b0]);
 
 *ecx &= hvm_featureset[FEATURESET_7c0];
+*edx &= hvm_featureset[FEATURESET_7d0];
 
 /* Don't expose HAP-only features to non-hap guests. */
 if ( !hap_enabled(d) )
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index 14abb62..2469e49 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -1128,6 +1128,7 @@ void pv_cpuid(struct cpu_user_regs *regs)
   special_features[FEATURESET_7b0]);
 
 c &= pv_featureset[FEATURESET_7c0];
+d &= pv_featureset[FEATURESET_7d0];
 
 if ( !is_pvh_domain(currd) )
 {
@@ -1142,8 +1143,8 @@ void pv_cpuid(struct cpu_user_regs *regs)
 }
 }
 else
-b = c = 0;
-a = d = 0;
+b = c = d = 0;
+a = 0;
 break;
 
 case XSTATE_CPUID:
diff --git a/xen/include/asm-x86/cpuid.h b/xen/include/asm-x86/cpuid.h
index 2372474..ec8bbb5 100644
--- a/xen/include/asm-x86/cpuid.h
+++ b/xen/include/asm-x86/cpuid.h
@@ -17,6 +17,7 @@
 #define FEATURESET_7c06 /* 0x0007:0.ecx*/
 #define FEATURESET_e7d7 /* 0x8007.edx  */
 #define FEATURESET_e8b8 /* 0x8008.ebx  */
+#define FEATURESET_7d09 /* 0x0007:0.edx*/
 
 #ifndef __ASSEMBLY__
 #include 
diff --git a/xen/include/public/arch-x86/cpufeatureset.h 
b/xen/include/public/arch-x86/cpufeatureset.h
index 9320c9e..565ccd5 100644
--- a/xen/include/public/arch-x86/cpufeat

[Xen-devel] [PATCH v8] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

SMEP/SMAP is a security feature to prevent kernel executing/accessing
user address involuntarily, any such behavior will lead to a page fault.

SMEP/SMAP is open (in CR4) for both Xen and HVM guest in earlier code.
SMEP/SMAP bit set in Xen CR4 would enforce security checking for 32-bit
PV guest which will suffer unknown SMEP/SMAP page fault when guest
kernel attempt to access user address although SMEP/SMAP is close for
PV guests.

This patch introduces a new boot option value "hvm" for "sm{e,a}p", it
is going to diable SMEP/SMAP for Xen hypervisor while enable them for
HVM. In this way, 32-bit PV guest will not suffer SMEP/SMAP security
issue. Users can choose whether open SMEP/SMAP for Xen itself,
especially when they are going to run 32-bit PV guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>

---
Changes in v8:  
* code style.   
* test: this patch has been tested under 32-bit PV guest with and  
without hardware SMAP feature.  

Changes in v7:  
* bugfix: fix the bug that this patch doesn't work on machine without SMAP.
* test: This patch has not been tested (on 32-bit PV environment). 
Really sorry for that since I have took several days trying to  
setup a 32-bit PV guest but finally failed. 

Changes in v6:  
* fix sm{e,a}p parameters parser flow.  

Changes in v5:  
* refine sm{e,a}p parameters parser flow.   
* replace cpu_has_sm{e,a}p with boot_cpu_has(X86_FEATURE_XEN_SM{E,A}P). 
* refine docs.  

Changes in v4:  
* introduce 2 new synthetic features X86_FEATURE_XEN_SMEP and   
  X86_FEATURE_XEN_SMAP for Xen itself.  
* adjust SM{E,A}P related instruction patching code.
* commit message refinement.

Changes in v3:  
* fix boot options. 
* fix CR4 & mmu_cr4_features operations.
* disable SMEP/SMAP for Dom0.   
* commit message refinement.

Changes in v2:  
* allow "hvm" as a value to "smep" and "smap" command line options.
* clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only. 
* refine docs.  
* rewrite commit message.
---
 docs/misc/xen-command-line.markdown | 10 +++---
 xen/arch/x86/setup.c| 68 +++--
 xen/arch/x86/x86_64/compat/entry.S  |  4 +--
 xen/arch/x86/x86_64/entry.S |  4 +--
 xen/include/asm-x86/asm_defns.h | 10 +++---
 xen/include/asm-x86/cpufeature.h|  2 ++
 6 files changed, 75 insertions(+), 23 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index cd9534b..41e2116 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1433,19 +1433,21 @@ enabling more sockets and cores to go into deeper sleep 
states.
 
 Set the serial transmit buffer size.
 
-### smep
+### smap
 > `= `
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Execution Protection
+Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to allow SMAP use by HVM guests only.
 
-### smap
+### smep
 > `= `
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Access Prevention
+Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to allow SMEP use by HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 8ae897a..58b117d 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -61,14 +61,6 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
-/* smep: Enable/disable Supervisor Mode Exe

Re: [Xen-devel] [PATCH v7] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

On Mon, Oct 10, 2016 at 06:16:41AM -0600, Jan Beulich wrote:
> >>> On 09.10.16 at 10:20,  wrote:
> > Changes in v7:
> > * bugfix: fix the bug that this patch doesn't work on machine without SMAP.
> > * test: This patch has not been tested (on 32-bit PV environment).
> > Really sorry for that since I have took several days trying to
> > setup a 32-bit PV guest but finally failed.
> 
> Well, I don't know what to say. And since you don't say what your
> problem is/was, I also don't see how anyone could help.
> 
Apologies for sending out a patch without testing it entirely.
To be honest, I am not so fimilar with 32-bit PV guest...
I create a HVM guest and then compile linux kernel with the configuration
suggested by https://wiki.xen.org/wiki/Mainline_Linux_Kernel_Configs
after that, I copy vmlinuz and initramfs from guest to host and I boot
guest with PV cfg file below:
```
name="rhel32"
memory=2048
vcpus=2
on_crash="destroy"
on_poweroff="destroy"
on_reboot="restart"
localtime=0
builder="linux"
kernel="/root/xen_guest/rhel32pv/vmlinuz-4.8.0+"
ramdisk="/root/xen_guest/rhel32pv/initramfs-4.8.0+.img"
extra="root=/dev/xvda"
disk=['file:/root/xen_guest/rhel32pv/rhel32.img,xvda,w',]
vif=[ 'mac=00:15:3e:22:f5:1b','bridge=xenbr0']
```
The guest fail to boot and error message says can not find modules in
/lib/module...

I also try to install a PV guest directly by netboot way, but I am
blocked at "installing kernel" step, error shows and let me try.

The problems are trivial, and may I ask for some "how to build a 32-bit
PV guest on 64-bit host" reference wiki page or threads?

> > @@ -1404,12 +1448,16 @@ void __init noreturn __start_xen(unsigned long 
> > mbi_p)
> >  
> >  if ( !opt_smep )
> >  setup_clear_cpu_cap(X86_FEATURE_SMEP);
> > -if ( cpu_has_smep )
> > +else if ( cpu_has_smep && opt_smep == 1 )
> 
> How about
> 
> if ( cpu_has_smep && opt_smep != SMEP_HVM_ONLY )
> 
> (i.e. I dislike both the "else" and the hard-coded 1)? Or if you dislike
> this, then at least > 0 instead of == 1 please, or provide a #define
> just like you do for -1.

Sure, I would use `if ( cpu_has_smep && opt_smep != SMEP_HVM_ONLY )` in
next version.
BTW. In v6 patch, I missed `cpu_has_smep` and that's why this patch is
buggy on the machine without smap/smep hardware feature.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH v7] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

From: He Chao <he.c...@linux.intel.com>

SMEP/SMAP is a security feature to prevent kernel executing/accessing
user address involuntarily, any such behavior will lead to a page fault.

SMEP/SMAP is open (in CR4) for both Xen and HVM guest in earlier code.
SMEP/SMAP bit set in Xen CR4 would enforce security checking for 32-bit
PV guest which will suffer unknown SMEP/SMAP page fault when guest
kernel attempt to access user address although SMEP/SMAP is close for
PV guests.

This patch introduces a new boot option value "hvm" for "sm{e,a}p", it
is going to diable SMEP/SMAP for Xen hypervisor while enable them for
HVM. In this way, 32-bit PV guest will not suffer SMEP/SMAP security
issue. Users can choose whether open SMEP/SMAP for Xen itself,
especially when they are going to run 32-bit PV guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes in v7:
* bugfix: fix the bug that this patch doesn't work on machine without SMAP.
* test: This patch has not been tested (on 32-bit PV environment).
Really sorry for that since I have took several days trying to
setup a 32-bit PV guest but finally failed.

Changes in v6:
* fix sm{e,a}p parameters parser flow.

Changes in v5:
* refine sm{e,a}p parameters parser flow.
* replace cpu_has_sm{e,a}p with boot_cpu_has(X86_FEATURE_XEN_SM{E,A}P).
* refine docs.

Changes in v4:
* introduce 2 new synthetic features X86_FEATURE_XEN_SMEP and
  X86_FEATURE_XEN_SMAP for Xen itself.
* adjust SM{E,A}P related instruction patching code.
* commit message refinement.

Changes in v3:
* fix boot options.
* fix CR4 & mmu_cr4_features operations.
* disable SMEP/SMAP for Dom0.
* commit message refinement.

Changes in v2:
* allow "hvm" as a value to "smep" and "smap" command line options.
* clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* refine docs.
* rewrite commit message.
---
 docs/misc/xen-command-line.markdown | 10 +++---
 xen/arch/x86/setup.c| 72 ++---
 xen/include/asm-x86/asm_defns.h | 10 +++---
 xen/include/asm-x86/cpufeature.h|  2 ++
 4 files changed, 73 insertions(+), 21 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 8ff57fa..b06b6ac 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1433,19 +1433,21 @@ enabling more sockets and cores to go into deeper sleep 
states.
 
 Set the serial transmit buffer size.
 
-### smep
+### smap
 > `= `
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Execution Protection
+Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to allow SMAP use by HVM guests only.
 
-### smap
+### smep
 > `= `
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Access Prevention
+Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to allow SMEP use by HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 8ae897a..f3ed247 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -61,14 +61,6 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
-/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
-
-/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
-
 unsigned long __read_mostly cr4_pv32_mask;
 
 /* Boot dom0 in pvh mode */
@@ -112,6 +104,58 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
+#define SMEP_HVM_ONLY (-1)
+static s8 __initdata opt_smep = 1;
+static void __init parse_smep_param(char *s)
+{
+if ( !*s )
+{
+opt_smep = 1;
+return;
+}
+
+switch ( parse_bool(s) )
+{
+case 0:
+opt_smep = 0;
+return;
+case 1:
+opt_smep = 1;
+return;
+}
+
+if ( !strcmp(s, "hvm") )
+opt_smep = SMEP_HVM_ONLY;
+}
+custom_param("smep", parse_smep_param);
+
+/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
+#define SMAP_HVM_ONLY (-1)
+static s8 __initdata opt_smap = 1;
+static void __init parse_smap_param(char *s)
+{
+if ( !*s )
+{
+opt_smap = 1;
+return;
+}
+
+switch ( parse_bool(s) )
+{
+case 0:
+opt_smap = 0;
+return;
+case 1:
+opt_smap = 1;
+return;
+}
+
+if ( !strcmp(s, "hvm") )
+opt_smap = SMAP_HVM_ONLY;
+}
+custom_param("smap", parse_smap_param);
+
 bool_t __read_mostly 

Re: [Xen-devel] [PATCH v6] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

On Tue, Sep 20, 2016 at 12:53:32AM -0600, Jan Beulich wrote:
> >>> On 20.09.16 at 04:29,  wrote:
> > Sorry for the late response, I saw the this patch was merged but soon
> > got reverted, and the revert message says this patch is still buggy.
> > 
> > I would be most grateful if you would point out the buggy part of this
> > patch and the reason why revert it.
> 
> Well, I've already told you on the 5th - see
> https://lists.xenproject.org/archives/html/xen-devel/2016-09/msg00414.html
> suggesting that you never tested the patch on older hardware.
> Additionally you could (and perhaps should) have looked yourself
> immediately at the failed smoke test logs (they may have got purged
> by now), referenced from the respective osstest mail
> https://lists.xenproject.org/archives/html/xen-devel/2016-09/msg00411.html

Oops, I didn't receive your reply at 5th Sep due to our mail server
temporarily down that day, terribly sorry..
Thank you for replying again and I will cook a patch and test it asap...

___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v6] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

Hi Jan,

Sorry for the late response, I saw the this patch was merged but soon
got reverted, and the revert message says this patch is still buggy.

I would be most grateful if you would point out the buggy part of this
patch and the reason why revert it.

Thanks,
-He

___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH v6] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

SMEP/SMAP is a security feature to prevent kernel executing/accessing
user address involuntarily, any such behavior will lead to a page fault.

SMEP/SMAP is open (in CR4) for both Xen and HVM guest in earlier code.
SMEP/SMAP bit set in Xen CR4 would enforce security checking for 32-bit
PV guest which will suffer unknown SMEP/SMAP page fault when guest
kernel attempt to access user address although SMEP/SMAP is close for
PV guests.

This patch introduces a new boot option value "hvm" for "sm{e,a}p", it
is going to diable SMEP/SMAP for Xen hypervisor while enable them for
HVM. In this way, 32-bit PV guest will not suffer SMEP/SMAP security
issue. Users can choose whether open SMEP/SMAP for Xen itself,
especially when they are going to run 32-bit PV guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>

---
Changes in v6:
* fix sm{e,a}p parameters parser flow.

Changes in v5:
* refine sm{e,a}p parameters parser flow.
* replace cpu_has_sm{e,a}p with boot_cpu_has(X86_FEATURE_XEN_SM{E,A}P).
* refine docs.

Changes in v4:
* introduce 2 new synthetic features X86_FEATURE_XEN_SMEP and
  X86_FEATURE_XEN_SMAP for Xen itself.
* adjust SM{E,A}P related instruction patching code.
* commit message refinement.

Changes in v3:
* fix boot options.
* fix CR4 & mmu_cr4_features operations.
* disable SMEP/SMAP for Dom0.
* commit message refinement.

Changes in v2:
* allow "hvm" as a value to "smep" and "smap" command line options.
* clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* refine docs.
* rewrite commit message.
---
 docs/misc/xen-command-line.markdown |  2 +
 xen/arch/x86/setup.c| 76 +++--
 xen/include/asm-x86/asm_defns.h | 10 ++---
 xen/include/asm-x86/cpufeature.h|  4 +-
 4 files changed, 73 insertions(+), 19 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..0225974 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1433,6 +1433,7 @@ Set the serial transmit buffer size.
 > Default: `true`
 
 Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to allow SMEP use by HVM guests only.
 
 ### smap
 > `= `
@@ -1440,6 +1441,7 @@ Flag to enable Supervisor Mode Execution Protection
 > Default: `true`
 
 Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to allow SMAP use by HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..3c41715 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -61,14 +61,6 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
-/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
-
-/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
-
 unsigned long __read_mostly cr4_pv32_mask;
 
 /* Boot dom0 in pvh mode */
@@ -111,6 +103,62 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
+#define SMEP_HVM_ONLY (-1)
+static s8 __initdata opt_smep = 1;
+static void __init parse_smep_param(char *s)
+{
+if ( !*s )
+{
+opt_smep = 1;
+return;
+}
+
+switch ( parse_bool(s) )
+{
+case 0:
+opt_smep = 0;
+return;
+case 1:
+opt_smep = 1;
+return;
+}
+
+if ( !strcmp(s, "hvm") )
+{
+opt_smep = SMEP_HVM_ONLY;
+}
+}
+custom_param("smep", parse_smep_param);
+
+/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
+#define SMAP_HVM_ONLY (-1)
+static s8 __initdata opt_smap = 1;
+static void __init parse_smap_param(char *s)
+{
+if ( !*s )
+{
+opt_smap = 1;
+return;
+}
+
+switch ( parse_bool(s) )
+{
+case 0:
+opt_smap = 0;
+return;
+case 1:
+opt_smap = 1;
+return;
+}
+
+if ( !strcmp(s, "hvm") )
+{
+opt_smap = SMAP_HVM_ONLY;
+}
+}
+custom_param("smap", parse_smap_param);
+
 bool_t __read_mostly acpi_disabled;
 bool_t __initdata acpi_force;
 static char __initdata acpi_param[10] = "";
@@ -1403,12 +1451,16 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
-if ( cpu_has_smep )
+else if ( opt_smep == 1 )
+__set_bit(X86_FEATURE_XEN_SMEP, boot_cpu_data.x86_capability);
+if ( boot_cpu_has(X86_FEATURE_XEN_SMEP) )
 set_in_cr4

[Xen-devel] [PATCH v5] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

SMEP/SMAP is a security feature to prevent kernel executing/accessing
user address involuntarily, any such behavior will lead to a page fault.

SMEP/SMAP is open (in CR4) for both Xen and HVM guest in earlier code.
SMEP/SMAP bit set in Xen CR4 would enforce security checking for 32-bit
PV guest which will suffer unknown SMEP/SMAP page fault when guest
kernel attempt to access user address although SMEP/SMAP is close for
PV guests.

This patch introduces a new boot option value "hvm" for "sm{e,a}p", it
is going to diable SMEP/SMAP for Xen hypervisor while enable them for
HVM. In this way, 32-bit PV guest will not suffer SMEP/SMAP security
issue. Users can choose whether open SMEP/SMAP for Xen itself,
especially when they are going to run 32-bit PV guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>

---
Changes in v5:
* refine sm{e,a}p parameters parser flow.
* replace cpu_has_sm{e,a}p with boot_cpu_has(X86_FEATURE_XEN_SM{E,A}P).
* refine docs.

Changes in v4:
* introduce 2 new synthetic features X86_FEATURE_XEN_SMEP and
  X86_FEATURE_XEN_SMAP for Xen itself.
* adjust SM{E,A}P related instruction patching code.
* commit message refinement.

Changes in v3:
* fix boot options.
* fix CR4 & mmu_cr4_features operations.
* disable SMEP/SMAP for Dom0.
* commit message refinement.

Changes in v2:
* allow "hvm" as a value to "smep" and "smap" command line options.
* clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* refine docs.
* rewrite commit message.
---
 docs/misc/xen-command-line.markdown |  2 ++
 xen/arch/x86/setup.c| 54 -
 xen/include/asm-x86/asm_defns.h | 10 +++
 xen/include/asm-x86/cpufeature.h|  4 +--
 4 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..0225974 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1433,6 +1433,7 @@ Set the serial transmit buffer size.
 > Default: `true`
 
 Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to allow SMEP use by HVM guests only.
 
 ### smap
 > `= `
@@ -1440,6 +1441,7 @@ Flag to enable Supervisor Mode Execution Protection
 > Default: `true`
 
 Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to allow SMAP use by HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..5256e99 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -61,14 +61,6 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
-/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
-
-/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
-
 unsigned long __read_mostly cr4_pv32_mask;
 
 /* Boot dom0 in pvh mode */
@@ -111,6 +103,44 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
+#define SMEP_HVM_ONLY (-1)
+static s8 __initdata opt_smep = 1;
+static void __init parse_smep_param(char *s)
+{
+if ( !parse_bool(s) )
+{
+opt_smep = 0;
+}
+else if ( !strcmp(s, "hvm") )
+{
+opt_smep = SMEP_HVM_ONLY;
+}
+
+if ( opt_smep == 1 )
+__set_bit(X86_FEATURE_XEN_SMEP, boot_cpu_data.x86_capability);
+}
+custom_param("smep", parse_smep_param);
+
+/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
+#define SMAP_HVM_ONLY (-1)
+static s8 __initdata opt_smap = 1;
+static void __init parse_smap_param(char *s)
+{
+if ( !parse_bool(s) )
+{
+opt_smap = 0;
+}
+else if ( !strcmp(s, "hvm") )
+{
+opt_smap = SMAP_HVM_ONLY;
+}
+
+if ( opt_smap == 1 )
+__set_bit(X86_FEATURE_XEN_SMAP, boot_cpu_data.x86_capability);
+}
+custom_param("smap", parse_smap_param);
+
 bool_t __read_mostly acpi_disabled;
 bool_t __initdata acpi_force;
 static char __initdata acpi_param[10] = "";
@@ -1403,12 +1433,12 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
-if ( cpu_has_smep )
+if ( boot_cpu_has(X86_FEATURE_XEN_SMEP) )
 set_in_cr4(X86_CR4_SMEP);
 
 if ( !opt_smap )
 setup_clear_cpu_cap(X86_FEATURE_SMAP);
-if ( cpu_has_smap )
+if ( boot_cpu_has(X86_FEATURE_XEN_SMAP) )
 set_in_cr4(X86_CR4_SMAP);
 
 cr4_pv32_mask = mmu_cr4_features & XEN_CR4_PV32_BITS;

[Xen-devel] [PATCH v4] xen/sm{e, a}p: allow disabling sm{e, a}p for Xen itself

[He Chen]

SMEP/SMAP is a security feature to prevent kernel executing/accessing
user address involuntarily, any such behavior will lead to a page fault.

SMEP/SMAP is open (in CR4) for both Xen and HVM guest in earlier code.
SMEP/SMAP bit set in Xen CR4 would enforce security checking for 32-bit
PV guest which will suffer unknown SMEP/SMAP page fault when guest
kernel attempt to access user address although SMEP/SMAP is close for
PV guests.

This patch introduces a new boot option value "hvm" for "sm{e,a}p", it
is going to diable SMEP/SMAP for Xen hypervisor while enable them for
HVM. In this way, 32-bit PV guest will not suffer SMEP/SMAP security
issue. Users can choose whether open SMEP/SMAP for Xen itself,
especially when they are going to run 32-bit PV guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>

---
Changes in v4:
* introduce 2 new synthetic features X86_FEATURE_XEN_SMEP and
  X86_FEATURE_XEN_SMAP for Xen itself.
* adjust SM{E,A}P related instruction patching code.
* Commit message refinement.

Changes in v3:
* Fix boot options.
* Fix CR4 & mmu_cr4_features operations.
* Disable SMEP/SMAP for Dom0.
* Commit message refinement.

Changes in v2:
* Allow "hvm" as a value to "smep" and "smap" command line options.
* Clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* Refine docs.
* Rewrite commit message.
---
 docs/misc/xen-command-line.markdown |  2 ++
 xen/arch/x86/setup.c| 58 -
 xen/include/asm-x86/asm_defns.h | 10 +++
 xen/include/asm-x86/cpufeature.h|  5 +++-
 4 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..b15f3e7 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1433,6 +1433,7 @@ Set the serial transmit buffer size.
 > Default: `true`
 
 Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to enable SMEP for HVM guests only.
 
 ### smap
 > `= `
@@ -1440,6 +1441,7 @@ Flag to enable Supervisor Mode Execution Protection
 > Default: `true`
 
 Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to enable SMAP for HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..59238a4 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -62,12 +62,12 @@ static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
 /* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
+static void parse_smep_param(char *s);
+custom_param("smep", parse_smep_param);
 
 /* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
+static void parse_smap_param(char *s);
+custom_param("smap", parse_smap_param);
 
 unsigned long __read_mostly cr4_pv32_mask;
 
@@ -111,6 +111,48 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+#define SMEP_HVM_ONLY (-1)
+static int __initdata opt_smep = 1;
+static void __init parse_smep_param(char *s)
+{
+if ( !strcmp(s, "hvm") )
+{
+opt_smep = SMEP_HVM_ONLY;
+}
+else if ( !parse_bool(s) )
+{
+opt_smep = 0;
+}
+else if ( parse_bool(s) && opt_smep != SMEP_HVM_ONLY )
+{
+opt_smep = 1;
+}
+
+if ( opt_smep == 1 )
+__set_bit(X86_FEATURE_XEN_SMEP, boot_cpu_data.x86_capability);
+}
+
+#define SMAP_HVM_ONLY (-1)
+static int __initdata opt_smap = 1;
+static void __init parse_smap_param(char *s)
+{
+if ( !strcmp(s, "hvm") )
+{
+opt_smap = SMAP_HVM_ONLY;
+}
+else if ( !parse_bool(s) )
+{
+opt_smap = 0;
+}
+else if ( parse_bool(s) && opt_smap != SMAP_HVM_ONLY )
+{
+opt_smap = 1;
+}
+
+if ( opt_smap == 1 )
+__set_bit(X86_FEATURE_XEN_SMAP, boot_cpu_data.x86_capability);
+}
+
 bool_t __read_mostly acpi_disabled;
 bool_t __initdata acpi_force;
 static char __initdata acpi_param[10] = "";
@@ -1403,12 +1445,12 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
-if ( cpu_has_smep )
+if ( cpu_has_xen_smep )
 set_in_cr4(X86_CR4_SMEP);
 
 if ( !opt_smap )
 setup_clear_cpu_cap(X86_FEATURE_SMAP);
-if ( cpu_has_smap )
+if ( cpu_has_xen_smap )
 set_in_cr4(X86_CR4_SMAP);
 
 cr4_pv32_mask = mmu_cr4_features & XEN_CR4_PV32_BITS;
@@ -1550,7 +1592,7 @@ void __init noreturn __start_xen(unsigned long mbi_p)
  * This saves a large number of corner cases i

Re: [Xen-devel] [PATCH v3] xen: support enabling SMEP/SMAP for HVM only

[He Chen]

On Wed, Aug 24, 2016 at 04:01:53AM -0600, Jan Beulich wrote:
> >>> On 19.08.16 at 12:20,  wrote:
> > Changes in v3:
> > * Fix boot options.
> > * Fix CR4 & mmu_cr4_features operations.
> > * Disable SMEP/SMAP for Dom0.
> > * Commit message refinement.
> 
> Several of my comments on v3 did not get taken care of (neither in
> code nor verbally). I'm not going to repeat them here.
> 
Let me try to address them well in next patch...

> > @@ -1403,12 +1437,12 @@ void __init noreturn __start_xen(unsigned long 
> > mbi_p)
> >  
> >  if ( !opt_smep )
> >  setup_clear_cpu_cap(X86_FEATURE_SMEP);
> > -if ( cpu_has_smep )
> > +if ( cpu_has_smep && opt_smep != SMEP_HVM_ONLY )
> >  set_in_cr4(X86_CR4_SMEP);
> >  
> >  if ( !opt_smap )
> >  setup_clear_cpu_cap(X86_FEATURE_SMAP);
> > -if ( cpu_has_smap )
> > +if ( cpu_has_smap && opt_smap != SMAP_HVM_ONLY )
> >  set_in_cr4(X86_CR4_SMAP);
> 
> So this avoids setting the flags in CR4, but also in mmu_cr4_features.
> 
I am sorry that I am not so clear about this. As what I see in the code,
`mmu_cr4_features` get changed in `set_in_cr4` and `clear_in_cr4` only.
May I ask why the sm{e,a}p is also in `mmu_cr4_features` and where else
it will be set?

> > @@ -1430,8 +1464,19 @@ void __init noreturn __start_xen(unsigned long mbi_p)
> >  
> >  arch_init_memory();
> >  
> > +/*
> > + * Temporarily clear SMAP in internal feature bitmap to avoid
> > + * patching unnecessary SMAP instructions when SMAP is disabled in
> > + * Xen hypervisor.
> > + */
> > +if ( opt_smap == SMAP_HVM_ONLY )
> > +__clear_bit(X86_FEATURE_SMAP, boot_cpu_data.x86_capability);
> > +
> >  alternative_instructions();
> >  
> > +if ( opt_smap == SMAP_HVM_ONLY )
> > +__set_bit(X86_FEATURE_SMAP, boot_cpu_data.x86_capability);
> 
> I think the better approach would be to introduce a synthetic
> feature, which gets set only when SMAP gets used by Xen for
> itself. Even if not needed for alternative patching, I think for
> symmetry reasons the same should then also be done for SMEP.
> 
Here, do you suggest to add a artificial SMAP feature (not from hardware
but according to the `smap` option) bit in `x86_capability` and to patch
SMAP instruction according to this new bit rather than actual hardware
SMAP bit?

Regarding SMEP, even if there are not instructions need to be patched,
but for symmetry reasons we should also add **another** new SMEP bit in
`x86_capability`, right?

> > @@ -1098,6 +1099,12 @@ void pv_cpuid(struct cpu_user_regs *regs)
> >  b |= (host_featureset[FEATURESET_7b0] &
> >special_features[FEATURESET_7b0]);
> >  
> > +if ( opt_smep == SMEP_HVM_ONLY )
> > +b &= ~cpufeat_mask(X86_FEATURE_SMEP);
> > +
> > +if ( opt_smap == SMAP_HVM_ONLY )
> > +b &= ~cpufeat_mask(X86_FEATURE_SMAP);
> 
> While you changed the place where you do the adjustment, my
> previous comment holds: "These flags already can't be set in
> pv_featureset, so the change is pointless."
> 
My carelessness, sorry for this.

> > --- a/xen/include/asm-x86/setup.h
> > +++ b/xen/include/asm-x86/setup.h
> > @@ -51,6 +51,12 @@ void microcode_grab_module(
> >  
> >  extern uint8_t kbd_shift_flags;
> >  
> > +#define SMEP_HVM_ONLY -1
> > +extern int opt_smep;
> > +
> > +#define SMAP_HVM_ONLY -1
> > +extern int opt_smap;
> 
> Which then means that these still don't need to become non-static.
> The #define-s, if you mean to retain them (in setup.c) would of
> course need proper parenthesization.
> 
> Jan
> 
> 
> ___
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> https://lists.xen.org/xen-devel

___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH v3] xen: support enabling SMEP/SMAP for HVM only

[He Chen]

SMEP/SMAP is a security feature to prevent kernel executing/accessing
user address involuntarily, any such behavior will lead to a page fault.

SMEP/SMAP is open (in CR4) for both Xen and HVM guest in earlier code.
A 32-bit PV guest will suffer unknown SMEP/SMAP page fault when guest
kernel attempt to access user address although SMEP/SMAP is close for
PV guests already.

This patch is going to support enabling SMEP/SMAP for HVM but disabling
them for Xen hypervisor. Users can choose whether opening them for Xen,
especially when they are going to run 32-bit PV guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>

---
Changes in v3:
* Fix boot options.
* Fix CR4 & mmu_cr4_features operations.
* Disable SMEP/SMAP for Dom0.
* Commit message refinement.

Changes in v2:
* Allow "hvm" as a value to "smep" and "smap" command line options.
* Clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* Refine docs.
* Rewrite commit message.
---
 docs/misc/xen-command-line.markdown |  2 ++
 xen/arch/x86/setup.c| 61 -
 xen/arch/x86/traps.c|  7 +
 xen/include/asm-x86/setup.h |  6 
 4 files changed, 68 insertions(+), 8 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..b15f3e7 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1433,6 +1433,7 @@ Set the serial transmit buffer size.
 > Default: `true`
 
 Flag to enable Supervisor Mode Execution Protection
+Use `smep=hvm` to enable SMEP for HVM guests only.
 
 ### smap
 > `= `
@@ -1440,6 +1441,7 @@ Flag to enable Supervisor Mode Execution Protection
 > Default: `true`
 
 Flag to enable Supervisor Mode Access Prevention
+Use `smap=hvm` to enable SMAP for HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..a428558 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -62,12 +62,12 @@ static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
 /* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
+static void parse_smep_param(char *s);
+custom_param("smep", parse_smep_param);
 
 /* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
+static void parse_smap_param(char *s);
+custom_param("smap", parse_smap_param);
 
 unsigned long __read_mostly cr4_pv32_mask;
 
@@ -111,6 +111,40 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+int opt_smep = 1;
+static void __init parse_smep_param(char *s)
+{
+if ( !strcmp(s, "hvm") )
+{
+opt_smep = SMEP_HVM_ONLY;
+}
+else if ( !parse_bool(s) )
+{
+opt_smep = 0;
+}
+else if ( parse_bool(s) && opt_smep != SMEP_HVM_ONLY )
+{
+opt_smep = 1;
+}
+}
+
+int opt_smap = 1;
+static void __init parse_smap_param(char *s)
+{
+if ( !strcmp(s, "hvm") )
+{
+opt_smap = SMAP_HVM_ONLY;
+}
+else if ( !parse_bool(s) )
+{
+opt_smap = 0;
+}
+else if ( parse_bool(s) && opt_smap != SMAP_HVM_ONLY )
+{
+opt_smap = 1;
+}
+}
+
 bool_t __read_mostly acpi_disabled;
 bool_t __initdata acpi_force;
 static char __initdata acpi_param[10] = "";
@@ -1403,12 +1437,12 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
-if ( cpu_has_smep )
+if ( cpu_has_smep && opt_smep != SMEP_HVM_ONLY )
 set_in_cr4(X86_CR4_SMEP);
 
 if ( !opt_smap )
 setup_clear_cpu_cap(X86_FEATURE_SMAP);
-if ( cpu_has_smap )
+if ( cpu_has_smap && opt_smap != SMAP_HVM_ONLY )
 set_in_cr4(X86_CR4_SMAP);
 
 cr4_pv32_mask = mmu_cr4_features & XEN_CR4_PV32_BITS;
@@ -1430,8 +1464,19 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
 arch_init_memory();
 
+/*
+ * Temporarily clear SMAP in internal feature bitmap to avoid
+ * patching unnecessary SMAP instructions when SMAP is disabled in
+ * Xen hypervisor.
+ */
+if ( opt_smap == SMAP_HVM_ONLY )
+__clear_bit(X86_FEATURE_SMAP, boot_cpu_data.x86_capability);
+
 alternative_instructions();
 
+if ( opt_smap == SMAP_HVM_ONLY )
+__set_bit(X86_FEATURE_SMAP, boot_cpu_data.x86_capability);
+
 local_irq_enable();
 
 pt_pci_init();
@@ -1550,7 +1595,7 @@ void __init noreturn __start_xen(unsigned long mbi_p)
  * This saves a large number of corner cases interactions with
  * copy_from_user().
  */
-   

Re: [Xen-devel] [PATCH V2] xen: support enabling SMEP/SMAP for HVM only

[He Chen]

On Thu, Aug 11, 2016 at 07:14:06AM -0600, Jan Beulich wrote:
> >>> On 11.08.16 at 11:17,  wrote:
> > @@ -1404,12 +1438,20 @@ void __init noreturn __start_xen(unsigned long 
> > mbi_p)
> >  if ( !opt_smep )
> >  setup_clear_cpu_cap(X86_FEATURE_SMEP);
> >  if ( cpu_has_smep )
> > +{
> >  set_in_cr4(X86_CR4_SMEP);
> > +if ( smep_hvm_only )
> > +write_cr4(read_cr4() & ~X86_CR4_SMEP);
> > +}
> 
> So that'll clear CR4.SMEP right here, but won't help with CR4 loads
> from mmu_cr4_features (as e.g. happens indirectly during VM exits,
> since the HOST_CR4 field gets set from this variable).
> 
> Did you in fact test your change, including validation of the features
> correctly remaining off over the lifetime of the system?
> 
> Further, considering that you don't clear the two flags from Xen's
> internal feature bitmap, and taken together with the internal feature
> bitmap driving alternative instruction patching, I'd assume pointless
> (and performance wise perhaps harmful) patching to now take place.
> 
Let me see whether I understand this correctly...

Regarding alternative instruction patching, if enabling SMAP for HVM but
disabling it for Xen, SMAP bit must be set in x86_capability feature
bitmap and cleared in mmu_cr4_features, which means instruction patching
would take place and a #UD may occur (since SMAP is disable in Xen, but
STAC/CLAC are patched and called).

A little dirty solution I can think of now is to temperarily clear SMAP
bit in x86_capability before patching instruction and then set it back
when instruction patching finish, like:

```
if ( opt_smap == SMAP_HVM_ONLY )
setup_clear_cpu_cap(X86_FEATURE_SMAP);

alternative_instructions();

if ( opt_smap == SMAP_HVM_ONLY )
__set_bit(X86_FEATURE_SMAP, boot_cpu_data.x86_capability);
```

Appreciate it if you have a better solution.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH V2] xen: support enabling SMEP/SMAP for HVM only

[He Chen]

Enhance "skaj...@intel.com>mep" and "smap" command line options to support 
enabling SMEP
or SMAP for HVM only with allowing "hvm" as a value.

Signed-off-by: He Chen <he.c...@linux.intel.com>

---
Changes in V2:
* Allow "hvm" as a value to "smep" and "smap" command line options.
* Clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* Refine docs.
* Rewrite commit message.
---
 docs/misc/xen-command-line.markdown | 14 +
 xen/arch/x86/cpuid.c|  5 
 xen/arch/x86/setup.c| 58 -
 xen/include/asm-x86/setup.h |  3 ++
 4 files changed, 66 insertions(+), 14 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..0e49358 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1427,19 +1427,21 @@ enabling more sockets and cores to go into deeper sleep 
states.
 
 Set the serial transmit buffer size.
 
-### smep
-> `= `
+### smap
+> `=  | hvm`
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Execution Protection
+Flag to enable Supervisor Mode Access Prevention
+Using `smap=hvm` to enable SMAP for HVM guests only.
 
-### smap
-> `= `
+### smep
+> `=  | hvm`
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Access Prevention
+Flag to enable Supervisor Mode Execution Protection
+Using `smep=hvm` to enable SMEP for HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c
index 38e34bd..afa16b8 100644
--- a/xen/arch/x86/cpuid.c
+++ b/xen/arch/x86/cpuid.c
@@ -4,6 +4,7 @@
 #include 
 #include 
 #include 
+#include 
 
 const uint32_t known_features[] = INIT_KNOWN_FEATURES;
 const uint32_t special_features[] = INIT_SPECIAL_FEATURES;
@@ -118,6 +119,10 @@ static void __init calculate_pv_featureset(void)
 __set_bit(X86_FEATURE_HTT, pv_featureset);
 __set_bit(X86_FEATURE_X2APIC, pv_featureset);
 __set_bit(X86_FEATURE_CMP_LEGACY, pv_featureset);
+if ( smep_hvm_only )
+__clear_bit(X86_FEATURE_SMEP, pv_featureset);
+if ( smap_hvm_only )
+__clear_bit(X86_FEATURE_SMAP, pv_featureset);
 
 sanitise_featureset(pv_featureset);
 }
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..625b9b4 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -61,13 +61,19 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
-/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
-
-/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
+/* Supervisor Mode Execution Protection (default on). */
+/* "smep=on": Enable SMEP for Xen and guests. */
+/* "smep=hvm": Enable SMEP for HVM only.  */
+/* "smep=off": Disable SMEP for Xen and guests.   */
+static void parse_smep_param(char *s);
+custom_param("smep", parse_smep_param);
+
+/* Supervisor Mode Access Prevention (default on). */
+/* "smep=on": Enable SMAP for Xen and guests.  */
+/* "smep=hvm": Enable SMAP for HVM only.   */
+/* "smep=off": Disable SMAP for Xen and guests.*/
+static void parse_smap_param(char *s);
+custom_param("smap", parse_smap_param);
 
 unsigned long __read_mostly cr4_pv32_mask;
 
@@ -111,6 +117,34 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+static bool_t __initdata opt_smep = 1;
+bool_t __initdata smep_hvm_only = 0;
+static void __init parse_smep_param(char *s)
+{
+if ( !parse_bool(s) )
+{
+opt_smep = 0;
+}
+else if ( !strcmp(s, "hvm") )
+{
+smep_hvm_only = 1;
+}
+}
+
+static bool_t __initdata opt_smap = 1;
+bool_t __initdata smap_hvm_only = 0;
+static void __init parse_smap_param(char *s)
+{
+if ( !parse_bool(s) )
+{
+opt_smap = 0;
+}
+else if ( !strcmp(s, "hvm") )
+{
+smap_hvm_only = 1;
+}
+}
+
 bool_t __read_mostly acpi_disabled;
 bool_t __initdata acpi_force;
 static char __initdata acpi_param[10] = "";
@@ -1404,12 +1438,20 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
 if ( cpu_has_smep )
+{
 set_in_cr4(X86_CR4_SMEP);
+if ( smep_hvm_only )
+write_cr4(read_cr4() & ~X86_CR4_SMEP);
+}
 
 if ( !opt_smap )
 setup_clear_cpu_cap(X86_FEATURE_SMAP);
 if ( cpu_has_smap )
+{
 set_in_c

[Xen-devel] [PATCH V2] xen: support enabling SMEP/SMAP for HVM only

[He Chen]

Enhance "smep" and "smap" command line options to support enabling SMEP
or SMAP for HVM only with allowing "hvm" as a value.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes in V2:
* Allow "hvm" as a value to "smep" and "smap" command line options.
* Clear SMEP/SMAP CPUID bits for pv guests if they are set to hvm only.
* Refine docs.
* Rewrite commit message.
---
 docs/misc/xen-command-line.markdown | 14 +
 xen/arch/x86/cpuid.c|  5 
 xen/arch/x86/setup.c| 58 -
 xen/include/asm-x86/setup.h |  3 ++
 4 files changed, 66 insertions(+), 14 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..0e49358 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1427,19 +1427,21 @@ enabling more sockets and cores to go into deeper sleep 
states.
 
 Set the serial transmit buffer size.
 
-### smep
-> `= `
+### smap
+> `=  | hvm`
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Execution Protection
+Flag to enable Supervisor Mode Access Prevention
+Using `smap=hvm` to enable SMAP for HVM guests only.
 
-### smap
-> `= `
+### smep
+> `=  | hvm`
 
 > Default: `true`
 
-Flag to enable Supervisor Mode Access Prevention
+Flag to enable Supervisor Mode Execution Protection
+Using `smep=hvm` to enable SMEP for HVM guests only.
 
 ### snb\_igd\_quirk
 > `=  | cap | `
diff --git a/xen/arch/x86/cpuid.c b/xen/arch/x86/cpuid.c
index 38e34bd..afa16b8 100644
--- a/xen/arch/x86/cpuid.c
+++ b/xen/arch/x86/cpuid.c
@@ -4,6 +4,7 @@
 #include 
 #include 
 #include 
+#include 
 
 const uint32_t known_features[] = INIT_KNOWN_FEATURES;
 const uint32_t special_features[] = INIT_SPECIAL_FEATURES;
@@ -118,6 +119,10 @@ static void __init calculate_pv_featureset(void)
 __set_bit(X86_FEATURE_HTT, pv_featureset);
 __set_bit(X86_FEATURE_X2APIC, pv_featureset);
 __set_bit(X86_FEATURE_CMP_LEGACY, pv_featureset);
+if ( smep_hvm_only )
+__clear_bit(X86_FEATURE_SMEP, pv_featureset);
+if ( smap_hvm_only )
+__clear_bit(X86_FEATURE_SMAP, pv_featureset);
 
 sanitise_featureset(pv_featureset);
 }
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..625b9b4 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -61,13 +61,19 @@ boolean_param("nosmp", opt_nosmp);
 static unsigned int __initdata max_cpus;
 integer_param("maxcpus", max_cpus);
 
-/* smep: Enable/disable Supervisor Mode Execution Protection (default on). */
-static bool_t __initdata opt_smep = 1;
-boolean_param("smep", opt_smep);
-
-/* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
-static bool_t __initdata opt_smap = 1;
-boolean_param("smap", opt_smap);
+/* Supervisor Mode Execution Protection (default on). */
+/* "smep=on": Enable SMEP for Xen and guests. */
+/* "smep=hvm": Enable SMEP for HVM only.  */
+/* "smep=off": Disable SMEP for Xen and guests.   */
+static void parse_smep_param(char *s);
+custom_param("smep", parse_smep_param);
+
+/* Supervisor Mode Access Prevention (default on). */
+/* "smep=on": Enable SMAP for Xen and guests.  */
+/* "smep=hvm": Enable SMAP for HVM only.   */
+/* "smep=off": Disable SMAP for Xen and guests.*/
+static void parse_smap_param(char *s);
+custom_param("smap", parse_smap_param);
 
 unsigned long __read_mostly cr4_pv32_mask;
 
@@ -111,6 +117,34 @@ struct cpuinfo_x86 __read_mostly boot_cpu_data = { 0, 0, 
0, 0, -1 };
 
 unsigned long __read_mostly mmu_cr4_features = XEN_MINIMAL_CR4;
 
+static bool_t __initdata opt_smep = 1;
+bool_t __initdata smep_hvm_only = 0;
+static void __init parse_smep_param(char *s)
+{
+if ( !parse_bool(s) )
+{
+opt_smep = 0;
+}
+else if ( !strcmp(s, "hvm") )
+{
+smep_hvm_only = 1;
+}
+}
+
+static bool_t __initdata opt_smap = 1;
+bool_t __initdata smap_hvm_only = 0;
+static void __init parse_smap_param(char *s)
+{
+if ( !parse_bool(s) )
+{
+opt_smap = 0;
+}
+else if ( !strcmp(s, "hvm") )
+{
+smap_hvm_only = 1;
+}
+}
+
 bool_t __read_mostly acpi_disabled;
 bool_t __initdata acpi_force;
 static char __initdata acpi_param[10] = "";
@@ -1404,12 +1438,20 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
 if ( cpu_has_smep )
+{
 set_in_cr4(X86_CR4_SMEP);
+if ( smep_hvm_only )
+write_cr4(read_cr4() & ~X86_CR4_SMEP);
+}
 
 if ( !opt_smap )
 setup_clear_cpu_cap(X86_FEATURE_SMAP);
 if ( cpu_has_smap )
+{
 set_in_cr4(

[Xen-devel] [PATCH] xen: enable/disable SMAP/SMEP for Xen itself

[He Chen]

SMAP/SMEP may affect the 32-bit pv guests.
Users can determine whether turn SMAP/SMEP on for Xen hyperviosr when
running 32-bit pv guests.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/misc/xen-command-line.markdown | 14 ++
 xen/arch/x86/setup.c| 12 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index 3a250cb..a48c4aa 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1434,6 +1434,13 @@ Set the serial transmit buffer size.
 
 Flag to enable Supervisor Mode Execution Protection
 
+### xen_smep
+> `= `
+
+> Default: `true`
+
+Flag to enable SMEP for Xen itself
+
 ### smap
 > `= `
 
@@ -1441,6 +1448,13 @@ Flag to enable Supervisor Mode Execution Protection
 
 Flag to enable Supervisor Mode Access Prevention
 
+### xen_smap
+> `= `
+
+> Default: `true`
+
+Flag to enable SMAP for Xen itself
+
 ### snb\_igd\_quirk
 > `=  | cap | `
 
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 217c775..84debc3 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -65,10 +65,18 @@ integer_param("maxcpus", max_cpus);
 static bool_t __initdata opt_smep = 1;
 boolean_param("smep", opt_smep);
 
+/* xen_smep: Enable/disable SMEP for Xen itself (default on). */
+static bool_t __initdata opt_xen_smep = 1;
+boolean_param("xen_smep", opt_xen_smep);
+
 /* smap: Enable/disable Supervisor Mode Access Prevention (default on). */
 static bool_t __initdata opt_smap = 1;
 boolean_param("smap", opt_smap);
 
+/* xen_smap: Enable/disable SMAP for Xen itself (default on). */
+static bool_t __initdata opt_xen_smap = 1;
+boolean_param("xen_smap", opt_xen_smap);
+
 unsigned long __read_mostly cr4_pv32_mask;
 
 /* Boot dom0 in pvh mode */
@@ -1403,12 +1411,12 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
 if ( !opt_smep )
 setup_clear_cpu_cap(X86_FEATURE_SMEP);
-if ( cpu_has_smep )
+if ( cpu_has_smep && opt_xen_smep )
 set_in_cr4(X86_CR4_SMEP);
 
 if ( !opt_smap )
 setup_clear_cpu_cap(X86_FEATURE_SMAP);
-if ( cpu_has_smap )
+if ( cpu_has_smap && opt_xen_smap )
 set_in_cr4(X86_CR4_SMAP);
 
 cr4_pv32_mask = mmu_cr4_features & XEN_CR4_PV32_BITS;
-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

[Xen-devel] [PATCH] solve SMAP/SMEP issues with 32-bit pv guests

[He Chen]

This patch is going to solve SMAP/SMEP issues with 32-bit pv guests by
adding new xen command line options "xen_smap" and "xen_smep".

For the details, please see: 
https://lists.xen.org/archives/html/xen-devel/2016-06/msg03441.html

I am sorry that I don't have 32-bit PV environment to test this patch,
I would really appreciate it if someone could help verify this patch :)

He Chen (1):
  xen: enable/disable SMAP/SMEP for Xen itself

 docs/misc/xen-command-line.markdown | 14 ++
 xen/arch/x86/setup.c| 12 ++--
 2 files changed, 24 insertions(+), 2 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel

Re: [Xen-devel] [RFC Design Doc] Intel L2 Cache Allocation Technology (L2 CAT) Feature enabling

[He Chen]

On Fri, May 13, 2016 at 06:17:53PM +0200, Dario Faggioli wrote:
> On Fri, 2016-05-13 at 10:23 +0100, Andrew Cooper wrote:
> > On 13/05/16 09:55, Jan Beulich wrote:
> > > 
> > > But anyway, L2 or L3 - I can't see how this context switching would
> > > DTRT when there are vCPU-s of different domains on the same
> > > socket (or core, if L2s and MSRs were per-core): The one getting
> > > scheduled later onto a socket (core) would simply overwrite what
> > > got written for the one which had been scheduled earlier.
> > PSR_ASSOC is a per-thread MSR which selects the CLOS to use.  CLOS is
> > currently managed per-domain in Xen, and context switched with vcpu.
> > 
> Yep, exactly. I did look a bit into this for CMT (so, not L3 CAT, but
> it's not that different).
> 
> Doing things on a per-vcpu basis could be interesting, and that's even
> more the case if we get to do L2 stuff, but there are two few RMIDs
> available for such a configuration to be really useful.
> 
> > Xen programs different capacity bitmaps into IA32_L2_QOS_MASK_0 ...
> > IA32_L2_QOS_MASK_n, and the CLOS selects which bitmap is enforced.
> > 
> So, basically, just to figure out if I understood (i.e., this is for He
> Chen).
> 
> If we have a 2 sockets, with socket 0 containing cores 0,1,2,3 and
> socket 1 containing cores 4,5,6,7, it will be possible to specify two
> different "L2 reservation values" (in the form of CBMs, of course), for
> a domain:
>  - one would be how much L2 cache the domain will be able to use (say 
>    X) when running on socket 1, which means on cores 0,1,2 or 3
>  - another would be how much L2 cache the domain will be able to (say, 
>    Y use when running on socket 2, which means on cores 4,5,6, or 7
> 
> Which in turn means, in case L2 is per core, the domain will get X of
> core 0's L2, X of core 1's L2, X of core 2's L2 and X of core 3's L2.
> On socket 1, it will get Y of core 4' L2, Y of core 5's L2 cache etc.
> etc.
> 
> And so, in summary what we would not be able to specify is a different
> value for the L2 reservations of, for instance, core 1 and core 3
> (i.e., of cores that are part of the same socket).
> 
> Does this summary make sense?

Yes, great instance and that is exactly how L3 CAT work now.
Let's see the source to make it clear:
```
void psr_ctxt_switch_to(struct domain *d)
{
...
if ( psra->cos_mask )
psr_assoc_cos(, d->arch.psr_cos_ids ?
  d->arch.psr_cos_ids[cpu_to_socket(smp_processor_id())] :
  0, psra->cos_mask);
...
}
```
`psr_cos_ids` is indexed by socket_id which leads to the per-socket
cache enforcement.

As Andrew said, CLOS is currently managed per-domain in Xen and it works
well so far. So in initial design, I am inclined to continue this
behavior (per-socket) to L2 CAT to keep the consistency between L2 and
L3 CAT. Any thoughts?

Thanks,
-He


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [RFC Design Doc] Intel L2 Cache Allocation Technology (L2 CAT) Feature enabling

[He Chen]

On Thu, May 12, 2016 at 04:05:36AM -0600, Jan Beulich wrote:
> >>> On 12.05.16 at 11:40,  wrote:
> > % Intel L2 Cache Allocation Technology (L2 CAT) Feature
> > % Revision 1.0
> > 
> > \clearpage
> > 
> > Hi all,
> > 
> > We plan to bring new PQoS feature called Intel L2 Cache Allocation
> > Technology (L2 CAT) to Xen.
> > 
> > L2 CAT is supported on Atom codename Goldmont and beyond. “Big-core”
> > Xeon does not support L2 CAT in current generations.
> 
> Looks mostly like a direct (and hence reasonable) extension of what
> we have for L3 right now. One immediate question I have is whether
> tying this to per-socket information is a good idea. As soon as Xeon-s
> would also gain such functionality, things would (aiui) need to become
> per-core (as L2 is per core there iirc).
> 

L2 Cache capability keeps the same through all cores in a socket, so we
make it per-socket to balance code complexity and accessibility.

I am not a expert in scheduler, do you mean in some cases, a domain
would apply different L2 cache access pattern when it is scheduled on
different cores even though the cores are in the same socket?

> The other question is whether with Xen we care enough about Atoms
> to add code that's of use only there.
> 

L2 CAT is a platform independent feature although it firstly shows in
Atoms and I believe that it will appear in other platform soon.

Thanks,
-He

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [RFC Design Doc] Intel L2 Cache Allocation Technology (L2 CAT) Feature enabling

[He Chen]

% Intel L2 Cache Allocation Technology (L2 CAT) Feature
% Revision 1.0

\clearpage

Hi all,

We plan to bring new PQoS feature called Intel L2 Cache Allocation
Technology (L2 CAT) to Xen.

L2 CAT is supported on Atom codename Goldmont and beyond. “Big-core”
Xeon does not support L2 CAT in current generations.

This is the initial design of L2 CAT. It might be a little long and
detailed, hope it doesn't matter.

Comments and suggestions are welcome :-)

# Basics

 
 Status: **Tech Preview**

Architecture(s): Intel x86

   Component(s): Hypervisor, toolstack

   Hardware: Atom codename Goldmont and beyond
 

# Overview

L2 CAT allows an OS or Hypervisor/VMM to control allocation of a
CPU's shared L2 cache based on application priority or Class of Service
(COS). Each CLOS is configured using capacity bitmasks (CBM) which
represent cache capacity and indicate the degree of overlap and
isolation between classes. Once L2 CAT is configured, the processor
allows access to portions of L2 cache according to the established
class of service (COS).

# Technical information

L2 CAT is a member of Intel PQoS features and part of CAT, it shares
some base PSR infrastructure in Xen.

## Hardware perspective

L2 CAT defines a new range MSRs to assign different L2 cache access
patterns which are known as CBMs (Capacity BitMask), each CBM is
associated with a COS.

```

+++
   IA32_PQR_ASSOC   | MSR (per socket)   |Address |
 ++---+---+ +++
 ||COS|   | | IA32_L2_QOS_MASK_0 | 0xD10  |
 ++---+---+ +++
└-> | ...|  ...   |
+++
| IA32_L2_QOS_MASK_n | 0xD10+n (n<64) |
+++
```

When context switch happens, the COS of VCPU is written to per-thread
MSR `IA32_PQR_ASSOC`, and then hardware enforces L2 cache allocation
according to the corresponding CBM.

## The relationship between L2 CAT and L3 CAT/CDP

L2 CAT is independent of L3 CAT/CDP, which means L2 CAT would be enabled
while L3 CAT/CDP is disabled, or L2 CAT and L3 CAT/CDP are all enabled.

L2 CAT uses a new range CBMs from 0xD10 ~ 0xD10+n (n<64), following by
the L3 CAT/CDP CBMs, and supports setting different L2 cache accessing
patterns from L3 cache.

N.B. L2 CAT and L3 CAT/CDP share the same COS field in the same
associate register `IA32_PQR_ASSOC`, that means one COS corresponds to a
pair of L2 CBM and L3 CBM.

In the initial implementation, L2 CAT is shown up on Atom codename
Goldmont firstly and there is no platform support both L2 & L3 CAT so
far.

## Design Overview

* Core COS/CBM association

  When enforcing L2 CAT, all cores of domains have the same default
  COS (COS0) which associated to the fully open CBM (all ones bitmask)
  to access all L2 cache. The default COS is used only in hypervisor
  and is transparent to tool stack and user.

  System administrator can change PQoS allocation policy at runtime by
  tool stack. Since L2 CAT share COS with L3 CAT/CDP, a COS corresponds
  to a 2-tuple, like [L2 CBM, L3 CBM] with only-CAT enabled, when CDP
  is enabled, one COS corresponds to a 3-tuple, like [L2 CBM,
  L3 Code_CBM, L3 Data_CBM]. If neither L3 CAT nor L3 CDP is enabled,
  things would be easier, one COS corresponds to one L2 CBM.

* VCPU schedule

  This part reuses L3 CAT COS infrastructure.

* Multi-sockets

  Different sockets may have different L2 CAT capability (e.g. max COS)
  although it is consistent on the same socket. So the capability of
  per-socket L2 CAT is specified.

## Implementation Description

* Hypervisor interfaces:

  1. Ext: Boot line parameter "psr=cat" now will enable L2 CAT and L3
  CAT if hardware supported.

  2. New: SYSCTL:
  - XEN_SYSCTL_PSR_CAT_get_l2_info: Get L2 CAT information.

  3. New: DOMCTL:
  - XEN_DOMCTL_PSR_CAT_OP_GET_L2_CBM: Get L2 CBM for a domain.
  - XEN_DOMCTL_PSR_CAT_OP_SET_L2_CBM: Set L2 CBM for a domain.

* xl interfaces:

  1. Ext: psr-cat-show: Show system/domain L2 CAT information.
  => XEN_SYSCTL_PSR_CAT_get_l2_info /
 XEN_DOMCTL_PSR_CAT_OP_GET_L2_CBM

  2. Ext: psr-mba-set -l2 domain-id cbm
  Set L2 cbm for a domain.
  => XEN_DOMCTL_PSR_CAT_OP_SET_L2_CBM

* Key data structure:

  1. Combined PSR bitmasks structure

 ```
 struct psr_mask {
 struct l3_cat {
 union {
 uint64_t cbm;
 struct {
 uint64_t code;
 uint64_t data;
 };
 

[Xen-devel] [RFC Design Doc] Intel L2 Cache Allocation Technology (L2 CAT) Feature enabling

[He Chen]

% Intel L2 Cache Allocation Technology (L2 CAT) Feature
% Revision 1.0

\clearpage

Hi all,

We plan to bring new PQoS feature called Intel L2 Cache Allocation
Technology (L2 CAT) to Xen.

L2 CAT is supported on Atom codename Goldmont and beyond. “Big-core”
Xeon does not support L2 CAT in current generations.

This is the initial design of L2 CAT. It might be a little long and
detailed, hope it doesn't matter.

Comments and suggestions are welcome :-)

# Basics

 
 Status: **Tech Preview**

Architecture(s): Intel x86

   Component(s): Hypervisor, toolstack

   Hardware: Atom codename Goldmont and beyond
 

# Overview

L2 CAT allows an OS or Hypervisor/VMM to control allocation of a
CPU's shared L2 cache based on application priority or Class of Service
(COS). Each CLOS is configured using capacity bitmasks (CBM) which
represent cache capacity and indicate the degree of overlap and
isolation between classes. Once L2 CAT is configured, the processor
allows access to portions of L2 cache according to the established
class of service (COS).

# Technical information

L2 CAT is a member of Intel PQoS features and part of CAT, it shares
some base PSR infrastructure in Xen.

## Hardware perspective

L2 CAT defines a new range MSRs to assign different L2 cache access
patterns which are known as CBMs (Capacity BitMask), each CBM is
associated with a COS.

```

+++
   IA32_PQR_ASSOC   | MSR (per socket)   |Address |
 ++---+---+ +++
 ||COS|   | | IA32_L2_QOS_MASK_0 | 0xD10  |
 ++---+---+ +++
└-> | ...|  ...   |
+++
| IA32_L2_QOS_MASK_n | 0xD10+n (n<64) |
+++
```

When context switch happens, the COS of VCPU is written to per-thread
MSR `IA32_PQR_ASSOC`, and then hardware enforces L2 cache allocation
according to the corresponding CBM.

## The relationship between L2 CAT and L3 CAT/CDP

L2 CAT is independent of L3 CAT/CDP, which means L2 CAT would be enabled
while L3 CAT/CDP is disabled, or L2 CAT and L3 CAT/CDP are all enabled.

L2 CAT uses a new range CBMs from 0xD10 ~ 0xD10+n (n<64), following by
the L3 CAT/CDP CBMs, and supports setting different L2 cache accessing
patterns from L3 cache.

N.B. L2 CAT and L3 CAT/CDP share the same COS field in the same
associate register `IA32_PQR_ASSOC`, that means one COS corresponds to a
pair of L2 CBM and L3 CBM.

In the initial implementation, L2 CAT is shown up on Atom codename
Goldmont firstly and there is no platform support both L2 & L3 CAT so
far.

## Design Overview

* Core COS/CBM association

  When enforcing L2 CAT, all cores of domains have the same default
  COS (COS0) which associated to the fully open CBM (all ones bitmask)
  to access all L2 cache. The default COS is used only in hypervisor
  and is transparent to tool stack and user.

  System administrator can change PQoS allocation policy at runtime by
  tool stack. Since L2 CAT share COS with L3 CAT/CDP, a COS corresponds
  to a 2-tuple, like [L2 CBM, L3 CBM] with only-CAT enabled, when CDP
  is enabled, one COS corresponds to a 3-tuple, like [L2 CBM,
  L3 Code_CBM, L3 Data_CBM]. If neither L3 CAT nor L3 CDP is enabled,
  things would be easier, one COS corresponds to one L2 CBM.

* VCPU schedule

  This part reuses L3 CAT COS infrastructure.

* Multi-sockets

  Different sockets may have different L2 CAT capability (e.g. max COS)
  although it is consistent on the same socket. So the capability of
  per-socket L2 CAT is specified.

## Implementation Description

* Hypervisor interfaces:

  1. Ext: Boot line parameter "psr=cat" now will enable L2 CAT and L3
  CAT if hardware supported.

  2. New: SYSCTL:
  - XEN_SYSCTL_PSR_CAT_get_l2_info: Get L2 CAT information.

  3. New: DOMCTL:
  - XEN_DOMCTL_PSR_CAT_OP_GET_L2_CBM: Get L2 CBM for a domain.
  - XEN_DOMCTL_PSR_CAT_OP_SET_L2_CBM: Set L2 CBM for a domain.

* xl interfaces:

  1. Ext: psr-cat-show: Show system/domain L2 CAT information.
  => XEN_SYSCTL_PSR_CAT_get_l2_info /
 XEN_DOMCTL_PSR_CAT_OP_GET_L2_CBM

  2. Ext: psr-mba-set -l2 domain-id cbm
  Set L2 cbm for a domain.
  => XEN_DOMCTL_PSR_CAT_OP_SET_L2_CBM

* Key data structure:

  1. Combined PSR bitmasks structure

 ```
 struct psr_mask {
 struct l3_cat {
 union {
 uint64_t cbm;
 struct {
 uint64_t code;
 uint64_t data;
 };
 

[Xen-devel] [PATCH v8] tools & docs: add tools and docs support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.
Add new CDP options with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
Reviewed-by: Chao Peng <chao.p.p...@linux.intel.com>
Acked-by: Ian Campbell <ian.campb...@citrix.com>
---
Changes in v8:
* fix per socekt CDP status show bug
* fix word missing in xl-psr.markdown
* coding style

Changes in v7:
* LIBXL_PSR_CBM_TYPE_L3_CODE (DATA) =>
  LIBXL_PSR_CBM_TYPE_L3_CBM_CODE (DATA)
* refactor psr_cat_print_one_domain and add a helper to print different
  type CBM
* fix docs wrap

Changes in v6:
* separate CBM headings in the output of xl psr-cat-show
* revert the numbers of SDM chapter in xl-psr.markdown
* XC_PSR_CAT_L3_CODE (DATA) => XC_PSR_CAT_L3_CBM_CODE (DATA)
* correct error message of passing -c and -d at the same time

Changes in v5:
* merge tools and docs patches
* replace EINVAL with ENXIO in libxl__psr_cat_log_err_msg
* revise options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
* refine CDP status output codes in psr_cat_hwinfo
* adjust CBM output format in command xl psr-cat-show
* docs revision

Example of new output format for command xl psr-cat-show:

*** CAT-only ***

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME CBM
0 Domain-0 0xf
1   centos.hvm 0xf

*** CDP enabled ***

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME  CBM (code) CBM (data)
0 Domain-0 0xf0xf
1   centos.hvm 0xf0xf
---
 docs/man/xl.pod.1 | 15 ++
 docs/misc/xl-psr.markdown | 53 ++
 tools/libxc/include/xenctrl.h |  7 +++--
 tools/libxc/xc_psr.c  | 17 ++-
 tools/libxl/libxl.h   |  7 +
 tools/libxl/libxl_psr.c   |  5 +++-
 tools/libxl/libxl_types.idl   |  3 ++
 tools/libxl/xl_cmdimpl.c  | 67 +++
 tools/libxl/xl_cmdtable.c |  2 ++
 9 files changed, 160 insertions(+), 16 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index d0cd612..4279c7c 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1518,6 +1518,13 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocations, which support specifying code or data cache for
+applications. CDP is used on a per VM basis in the Xen implementation. To
+specify code or data CBM for the domain, CDP feature must be enabled and CBM
+type options need to be specified when setting CBM, and the type options (code
+and data) are mutually exclusive.
+
 =over 4
 
 =item B [I] I I
@@ -1533,6 +1540,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM when CDP is enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM when CDP is enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index c32e25c..c3c1e8e 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -127,6 +127,59 @@ Per domain CBM settings can be shown by:
 
 `xl psr-cat-show`
 
+## Code and Data Prioritization (CDP)
+
+Code and Data Prioritization (CDP) Technology is an extension of CAT, which
+is available on Intel Broadwell and later server platforms. CDP enables
+isolation and separate prioritization of code and data fetches to the L3
+cache in a software configurable manner, which can enable workload
+prioritization and tuning of cache capacity to the characteristics of the
+workload. CDP extends Cache Allocation Technology (CAT) by providing
+separate code and data masks per Class of Service (COS).
+
+CDP can be enabled by adding `psr=cdp` to Xen command line.
+
+When CDP is enabled,
+
+ * the CAT masks are re-mapped into interleaved pairs of masks for data or
+   code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows the OS or Hypervisor to partition cache allocation in a more
+fine-grained manner. Code cache and data cache can be specified independently.
+With CDP enabled, one COS corresponds to two CBMs (code CBM & data CBM),
+since the sum of CBMs is fixed, that means the number of available COSes
+will reduce by half when CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"Platform Shared Resource Control: Cache Allocation Technology".
+
+T

[Xen-devel] [PATCH v8] detect and initialize CDP (Code/Data Prioritization) feature

[He Chen]

Changes in v8:
- x86:
  * x86/psr part is not included in this version since it had been merged
to upstream
- tools & docs:
  * fix per socekt CDP status show bug
  * fix word missing in xl-psr.markdown
  * coding style

Changes in v7:
- x86:
  * amend function find_cos and pick_avail_cos to ignore reference count
for COS0
  * write CDP data mask (mask1) before turning CDP on
  * check CDP status before clear enabled bit in function cat_cpu_fini
  * code style
- tools & docs:
  * LIBXL_PSR_CBM_TYPE_L3_CODE (DATA) =>
LIBXL_PSR_CBM_TYPE_L3_CBM_CODE (DATA)
  * refactor psr_cat_print_one_domain and add a helper to print different
type CBM
  * fix docs wrap

Changes in v6:
- x86:
  * remove unnecessary parameter in cdp_is_enabled function
  * remove variable need_write and restruct code in psr_set_l3_cbm
- tools & docs:
  * separate CBM headings in the output of xl psr-cat-show
  * revert the numbers of SDM chapter in xl-psr.markdown
  * XC_PSR_CAT_L3_CODE (DATA) => XC_PSR_CAT_L3_CBM_CODE (DATA)
  * correct error message of passing -c and -d at the same time

Changes in v5:
- x86: address Andrew's and Jan's coments.
- tools: refine options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
- merge tools and docs patches.
- code style

Changes in v4:
- x86:
  * remove union member name in struct `psr_cat_cbm` (suggested by Jan)
  * fix log info of CAT & CDP (suggested by Chao & Jan)
  * add a helper `cdp_is_enabled` to tell the status of CDP and CDP initialize
failed is considered (Jan's comment)
  * XEN_SYSCTL_INTERFACE_VERSION 0x000C -> 0x000D (suggested by Jan)
  * refine CBM type check logic in get/set CBM function (suggested by Jan)
  * loop optimization in function `find_cos` (suggested by Jan)
- tools: address Chao's comments.
- docs: address Chao's comments.
- code style

Changes in v3:
- x86: remove redundant CDP field in cat_socket_enable (suggested by Chao)
- tools: simplify CBM setting function in tools (suggested by Jan)
- docs: add boot parameter description (suggested by Chao & Ian)
- code style

Changes in v2:
- x86: enable CDP by boot parameter instead of enabling/disabling CDP at
   runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This v8 patch has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks.

He Chen (1):
  tools & docs: add tools and docs support for Intel CDP

 docs/man/xl.pod.1 | 15 ++
 docs/misc/xl-psr.markdown | 53 ++
 tools/libxc/include/xenctrl.h |  7 +++--
 tools/libxc/xc_psr.c  | 17 ++-
 tools/libxl/libxl.h   |  7 +
 tools/libxl/libxl_psr.c   |  5 +++-
 tools/libxl/libxl_types.idl   |  3 ++
 tools/libxl/xl_cmdimpl.c  | 67 +++
 tools/libxl/xl_cmdtable.c |  2 ++
 9 files changed, 160 insertions(+), 16 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v7 2/3] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

On Thu, Oct 15, 2015 at 09:47:37AM -0600, Jan Beulich wrote:
> 
> Ah, yes, in cases like this it should always be followed by return
> (or whatever else is suitable). Sorry for not having spotted this
> during review.
> 
Sorry for this bug. Is it proper to fix this bug by just adding a
return after ASSERT_UNREACHABLE? Or do some changes in
ASSERT_UNREACHABLE?

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v2 for staging] fix compile error

[He Chen]

In non-debug build ASSERT_UNREACHABLE is nop and some compilers will
complain that cbm_code/cbm_data may be used uninitialized in function
psr_set_l3_cbm. Add return after ASSERT_UNREACHABLE to fix it.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes in v2:
Sorry for mistake in first version of this patch.
This is the correct patch.
---
 xen/arch/x86/psr.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index d3dec3a..c5bdfce 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -477,6 +477,7 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket,
 
 default:
 ASSERT_UNREACHABLE();
+return -EINVAL;
 }
 
 spin_lock(>cbm_lock);
-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH for staging] fix compile error

[He Chen]

In non-debug build ASSERT_UNREACHABLE is nop and some compilers will
complain that cbm_code/cbm_data may be used uninitialized in function
psr_set_l3_cbm. Add return after ASSERT_UNREACHABLE to fix it.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/psr.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index d3dec3a..c5bdfce 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -477,6 +477,7 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket,
 
 default:
 ASSERT_UNREACHABLE();
+return;
 }
 
 spin_lock(>cbm_lock);
-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v7 0/3] detect and initialize CDP (Code/Data Prioritization) feature

[He Chen]

Changes in v7:
- x86:
  * amend function find_cos and pick_avail_cos to ignore reference count
for COS0
  * write CDP data mask (mask1) before turning CDP on
  * check CDP status before clear enabled bit in function cat_cpu_fini
  * code style
- tools & docs:
  * LIBXL_PSR_CBM_TYPE_L3_CODE (DATA) =>
LIBXL_PSR_CBM_TYPE_L3_CBM_CODE (DATA)
  * refactor psr_cat_print_one_domain and add a helper to print different
type CBM
  * fix docs wrap

Changes in v6:
- x86:
  * remove unnecessary parameter in cdp_is_enabled function
  * remove variable need_write and restruct code in psr_set_l3_cbm
- tools & docs:
  * separate CBM headings in the output of xl psr-cat-show
  * revert the numbers of SDM chapter in xl-psr.markdown
  * XC_PSR_CAT_L3_CODE (DATA) => XC_PSR_CAT_L3_CBM_CODE (DATA)
  * correct error message of passing -c and -d at the same time

Changes in v5:
- x86: address Andrew's and Jan's coments.
- tools: refine options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
- merge tools and docs patches.
- code style

Changes in v4:
- x86:
  * remove union member name in struct `psr_cat_cbm` (suggested by Jan)
  * fix log info of CAT & CDP (suggested by Chao & Jan)
  * add a helper `cdp_is_enabled` to tell the status of CDP and CDP initialize
failed is considered (Jan's comment)
  * XEN_SYSCTL_INTERFACE_VERSION 0x000C -> 0x000D (suggested by Jan)
  * refine CBM type check logic in get/set CBM function (suggested by Jan)
  * loop optimization in function `find_cos` (suggested by Jan)
- tools: address Chao's comments.
- docs: address Chao's comments.
- code style

Changes in v3:
- x86: remove redundant CDP field in cat_socket_enable (suggested by Chao)
- tools: simplify CBM setting function in tools (suggested by Jan)
- docs: add boot parameter description (suggested by Chao & Ian)
- code style

Changes in v2:
- x86: enable CDP by boot parameter instead of enabling/disabling CDP at
   runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This v7 patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks.

He Chen (3):
  x86: Support enable CDP by boot parameter and add get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools & docs: add tools and docs support for Intel CDP

 docs/man/xl.pod.1   |  15 +++
 docs/misc/xen-command-line.markdown |  11 +-
 docs/misc/xl-psr.markdown   |  53 
 tools/libxc/include/xenctrl.h   |   7 +-
 tools/libxc/xc_psr.c|  17 ++-
 tools/libxl/libxl.h |   7 ++
 tools/libxl/libxl_psr.c |   5 +-
 tools/libxl/libxl_types.idl |   3 +
 tools/libxl/xl_cmdimpl.c|  66 --
 tools/libxl/xl_cmdtable.c   |   2 +
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 232 +---
 xen/arch/x86/sysctl.c   |   5 +-
 xen/include/asm-x86/msr-index.h |   3 +
 xen/include/asm-x86/psr.h   |  20 +++-
 xen/include/public/domctl.h |   4 +
 xen/include/public/sysctl.h |   4 +-
 17 files changed, 414 insertions(+), 72 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v7 2/3] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

CDP extends CAT and provides the capacity to control L3 code & data
cache. With CDP, one COS corresponds to two CMBs(code & data). cbm_type
is added to distinguish different CBM operations. Besides, new domctl
cmds are introdunced to support set/get CDP CBM. Some CAT functions to
operation CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.coop...@citrix.com>
Reviewed-by: Chao Peng <chao.p.p...@linux.intel.com>
---
Changes in v7:
* amend function find_cos and pick_avail_cos to ignore reference count
  for COS0
* code style

Changes in v6:
* remove variable need_write and restruct code in psr_set_l3_cbm
* remove redundant type == PSR_CBM_TYPE_L3 in psr_get_l3_cbm

Changes in v5:
* replace -EINVAL with -ENXIO when setting code/data CBM on CDP
  disabled.
---
 xen/arch/x86/domctl.c   |  32 +++-
 xen/arch/x86/psr.c  | 181 ++--
 xen/include/asm-x86/psr.h   |  12 ++-
 xen/include/public/domctl.h |   4 +
 4 files changed, 183 insertions(+), 46 deletions(-)

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index f8a559c..0f6fdb9 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1168,12 +1168,40 @@ long arch_do_domctl(
 {
 case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
 ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
- domctl->u.psr_cat_op.data);
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 break;
 
 case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
 ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
- >u.psr_cat_op.data);
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 copyback = 1;
 break;
 
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 5ddeb2b..0083115 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -293,14 +293,40 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 return 0;
 }
 
-int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+int psr_get_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t *cbm, enum cbm_type type)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
+bool_t cdp_enabled = cdp_is_enabled(socket);
 
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+switch ( type )
+{
+case PSR_CBM_TYPE_L3:
+if ( cdp_enabled )
+return -EXDEV;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+break;
+
+case PSR_CBM_TYPE_L3_CODE:
+if ( !cdp_enabled )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].code;
+break;
+
+case PSR_CBM_TYPE_L3_DATA:
+if ( !cdp_enabled )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].data;
+break;
+
+default:
+ASSERT_UNREACHABLE();
+}
 
 return 0;
 }
@@ -331,19 +357,34 @@ static bool_t psr_check_cbm(unsigned int cbm_len, 
uint64_t cbm)
 struct cos_cbm_info
 {
 unsigned int cos;
-uint64_t cbm;
+uint64_t cbm_code;
+uint64_t cbm_data;
+bool_t cdp;
 };
 
 static void do_write_l3_cbm(void *data)
 {
 struct cos_cbm_info *info = data;
 
-wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
+if ( info->cdp )
+{
+wrmsrl(MSR_IA32_PSR_L3_MASK_CODE(info->cos), info->cbm_code);
+wrmsrl

[Xen-devel] [PATCH v7 1/3] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

Add boot parameter `psr=cdp` to enable CDP at boot time.
Intel Code/Data Prioritization (CDP) feature is based on CAT. Note that
cos_max would be half when CDP is on. struct psr_cat_cbm is extended to
support CDP operation. Extend psr_get_cat_l3_info sysctl to get CDP
status.

Signed-off-by: He Chen <he.c...@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.coop...@citrix.com>
---
Changes in v7:
* write CDP data mask (mask1) before turning CDP on
* check CDP status before clear enabled bit in function cat_cpu_fini

Changes in v6:
* remove unnecessary parameter in cdp_is_enabled

Changes in v5:
* remove unnecessary u in psr_cat_cbm structure
* revert write_l3_cbm and put the modification to next patch
* remove duplicate PSR_CAT_FLAG_L3_CDP
---
 docs/misc/xen-command-line.markdown | 11 ++--
 xen/arch/x86/psr.c  | 51 ++---
 xen/arch/x86/sysctl.c   |  5 ++--
 xen/include/asm-x86/msr-index.h |  3 +++
 xen/include/asm-x86/psr.h   |  8 +-
 xen/include/public/sysctl.h |  4 ++-
 6 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index a565c1b..416e559 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1174,9 +1174,9 @@ This option can be specified more than once (up to 8 
times at present).
 > `= `
 
 ### psr (Intel)
-> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: )`
+> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: | cdp: )`
 
-> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255`
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
 
 Platform Shared Resource(PSR) Services.  Intel Haswell and later server
 platforms offer information about the sharing of resources.
@@ -1206,6 +1206,13 @@ The following resources are available:
   the cache allocation.
   * `cat` instructs Xen to enable/disable Cache Allocation Technology.
   * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+  regarding the code cache and the data cache allocation. CDP is based on CAT.
+  * `cdp` instructs Xen to enable/disable Code and Data Prioritization. Note
+that `cos_max` of CDP is a little different from `cos_max` of CAT. With
+CDP, one COS will corespond two CBMs other than one with CAT, due to the
+sum of CBMs is fixed, that means actual `cos_max` in use will automatically
+reduce to half when CDP is enabled.
 
 ### reboot
 > `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | 
 > [c]old]`
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..5ddeb2b 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -21,9 +21,16 @@
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
+#define PSR_CDP(1<<2)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+};
+};
 unsigned int ref;
 };
 
@@ -43,6 +50,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_enable;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -94,6 +102,7 @@ static void __init parse_psr_param(char *s)
 
 parse_psr_bool(s, val_str, "cmt", PSR_CMT);
 parse_psr_bool(s, val_str, "cat", PSR_CAT);
+parse_psr_bool(s, val_str, "cdp", PSR_CDP);
 
 if ( val_str && !strcmp(s, "rmid_max") )
 opt_rmid_max = simple_strtoul(val_str, NULL, 0);
@@ -261,8 +270,13 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 return cat_socket_info + socket;
 }
 
+static inline bool_t cdp_is_enabled(unsigned int socket)
+{
+return cdp_socket_enable && test_bit(socket, cdp_socket_enable);
+}
+
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint32_t *flags)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +286,10 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
 
+*flags = 0;
+if ( cdp_is_enabled(socket) )
+*flags |= XEN_SYSCTL_PSR_CAT_L3_CDP;
+
 return 0;
 }
 
@@ -470,6 +488,7 @@ static void cat_cpu_init(void)
 struct psr_cat_socket_info *info;
 unsigned int socket;
 unsigned int cpu = smp_processor_id();
+uint64_t val;
 const struct cpuinfo_x86 *c = cpu_data + cpu;
 
 if ( !cpu_has(c, X86_FEATURE_CAT) || c->cpuid_level < PSR_CPUID_LEVEL_

[Xen-devel] [PATCH v7 3/3] tools & docs: add tools and docs support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.
Add new CDP options with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes in v7:
* LIBXL_PSR_CBM_TYPE_L3_CODE (DATA) =>
  LIBXL_PSR_CBM_TYPE_L3_CBM_CODE (DATA)
* refactor psr_cat_print_one_domain and add a helper to print different
  type CBM
* fix docs wrap

Changes in v6:
* separate CBM headings in the output of xl psr-cat-show
* revert the numbers of SDM chapter in xl-psr.markdown
* XC_PSR_CAT_L3_CODE (DATA) => XC_PSR_CAT_L3_CBM_CODE (DATA)
* correct error message of passing -c and -d at the same time

Changes in v5:
* merge tools and docs patches
* replace EINVAL with ENXIO in libxl__psr_cat_log_err_msg
* revise options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
* refine CDP status output codes in psr_cat_hwinfo
* adjust CBM output format in command xl psr-cat-show
* docs revision

Example of new output format for command xl psr-cat-show:

*** CAT-only ***

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME CBM
0 Domain-0 0xf
1   centos.hvm 0xf

*** CDP enabled ***

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME  CBM (code) CBM (data)
0 Domain-0 0xf0xf
1   centos.hvm 0xf0xf
---
 docs/man/xl.pod.1 | 15 ++
 docs/misc/xl-psr.markdown | 53 ++
 tools/libxc/include/xenctrl.h |  7 +++--
 tools/libxc/xc_psr.c  | 17 ++-
 tools/libxl/libxl.h   |  7 +
 tools/libxl/libxl_psr.c   |  5 +++-
 tools/libxl/libxl_types.idl   |  3 ++
 tools/libxl/xl_cmdimpl.c  | 66 +++
 tools/libxl/xl_cmdtable.c |  2 ++
 9 files changed, 159 insertions(+), 16 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index d0cd612..4279c7c 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1518,6 +1518,13 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocations, which support specifying code or data cache for
+applications. CDP is used on a per VM basis in the Xen implementation. To
+specify code or data CBM for the domain, CDP feature must be enabled and CBM
+type options need to be specified when setting CBM, and the type options (code
+and data) are mutually exclusive.
+
 =over 4
 
 =item B [I] I I
@@ -1533,6 +1540,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM when CDP is enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM when CDP is enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index c32e25c..9b08ee3 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -127,6 +127,59 @@ Per domain CBM settings can be shown by:
 
 `xl psr-cat-show`
 
+## Code and Data Prioritization (CDP)
+
+Code and Data Prioritization (CDP) Technology is an extension of CAT, which
+is available on Intel Broadwell and later server platforms. CDP enables
+isolation and separate prioritization of code and data fetches to the L3
+cache in a software configurable manner, which can enable workload
+prioritization and tuning of cache capacity to the characteristics of the
+workload. CDP extends Cache Allocation Technology (CAT) by providing
+separate code and data masks per Class of Service (COS).
+
+CDP can be enabled by adding `psr=cdp` to Xen command line.
+
+When CDP is enabled,
+
+ * the CAT masks are re-mapped into interleaved pairs of masks for data or
+   code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows the OS or Hypervisor to partition cache allocation in a more
+fine-grained. Code cache and data cache can be specified independently.
+With CDP enabled, one COS corresponds to two CBMs (code CBM & data CBM),
+since the sum of CBMs is fixed, that means the number of available COSes
+will reduce by half when CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"Platform Shared Resource Control: Cache Allocation Technology".
+
+The xl interfaces are the same with that of CAT. The difference is that
+CBM type can be passed as option to set code CBM or data CBM.
+
+When CDP is enabled, `-c` or `--code` option is available to set code CBM
+for the domai

[Xen-devel] [PATCH v6 0/3] detect and initialize CDP (Code/Data Prioritization) feature

[He Chen]

Changes in v6:
- x86:
  * remove unnecessary parameter in cdp_is_enabled function
  * remove variable need_write and restruct code in psr_set_l3_cbm
- tools & docs:
  * separate CBM headings in the output of xl psr-cat-show
  * revert the numbers of SDM chapter in xl-psr.markdown
  * XC_PSR_CAT_L3_CODE (DATA) => XC_PSR_CAT_L3_CBM_CODE (DATA)
  * correct error message of passing -c and -d at the same time

Changes in v5:
- x86: address Andrew's and Jan's coments.
- tools: refine options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
- merge tools and docs patches.
- code style

Changes in v4:
- x86:
  * remove union member name in struct `psr_cat_cbm` (suggested by Jan)
  * fix log info of CAT & CDP (suggested by Chao & Jan)
  * add a helper `cdp_is_enabled` to tell the status of CDP and CDP initialize
failed is considered (Jan's comment)
  * XEN_SYSCTL_INTERFACE_VERSION 0x000C -> 0x000D (suggested by Jan)
  * refine CBM type check logic in get/set CBM function (suggested by Jan)
  * loop optimization in function `find_cos` (suggested by Jan)
- tools: address Chao's comments.
- docs: address Chao's comments.
- code style

Changes in v3:
- x86: remove redundant CDP field in cat_socket_enable (suggested by Chao)
- tools: simplify CBM setting function in tools (suggested by Jan)
- docs: add boot parameter description (suggested by Chao & Ian)
- code style

Changes in v2:
- x86: enable CDP by boot parameter instead of enabling/disabling CDP at
   runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This v6 patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks.

He Chen (3):
  x86: Support enable CDP by boot parameter and add get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools & docs: add tools and docs support for Intel CDP

 docs/man/xl.pod.1   |  15 +++
 docs/misc/xen-command-line.markdown |  11 +-
 docs/misc/xl-psr.markdown   |  53 +
 tools/libxc/include/xenctrl.h   |   7 +-
 tools/libxc/xc_psr.c|  17 ++-
 tools/libxl/libxl.h |   7 ++
 tools/libxl/libxl_psr.c |   5 +-
 tools/libxl/libxl_types.idl |   3 +
 tools/libxl/xl_cmdimpl.c|  58 +++--
 tools/libxl/xl_cmdtable.c   |   2 +
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 229 
 xen/arch/x86/sysctl.c   |   5 +-
 xen/include/asm-x86/msr-index.h |   3 +
 xen/include/asm-x86/psr.h   |  20 +++-
 xen/include/public/domctl.h |   4 +
 xen/include/public/sysctl.h |   4 +-
 17 files changed, 404 insertions(+), 71 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v6 3/3] tools & docs: add tools and docs support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.
Add new CDP options with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes in v6:
* separate CBM headings in the output of xl psr-cat-show
* revert the numbers of SDM chapter in xl-psr.markdown
* XC_PSR_CAT_L3_CODE (DATA) => XC_PSR_CAT_L3_CBM_CODE (DATA)
* correct error message of passing -c and -d at the same time

Changes in v5:
* merge tools and docs patches
* replace EINVAL with ENXIO in libxl__psr_cat_log_err_msg
* revise options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
* refine CDP status output codes in psr_cat_hwinfo
* adjust CBM output format in command xl psr-cat-show
* docs revision

Example of new output format for command xl psr-cat-show:

*** CAT-only ***

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME CBM
0 Domain-0 0xf
1   centos.hvm 0xf

*** CDP enabled ***

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME  CBM (code) CBM (data)
0 Domain-0 0xf0xf
1   centos.hvm 0xf0xf
---
 docs/man/xl.pod.1 | 15 +++
 docs/misc/xl-psr.markdown | 53 +++
 tools/libxc/include/xenctrl.h |  7 --
 tools/libxc/xc_psr.c  | 17 -
 tools/libxl/libxl.h   |  7 ++
 tools/libxl/libxl_psr.c   |  5 +++-
 tools/libxl/libxl_types.idl   |  3 +++
 tools/libxl/xl_cmdimpl.c  | 58 +++
 tools/libxl/xl_cmdtable.c |  2 ++
 9 files changed, 152 insertions(+), 15 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index d0cd612..4279c7c 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1518,6 +1518,13 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocations, which support specifying code or data cache for
+applications. CDP is used on a per VM basis in the Xen implementation. To
+specify code or data CBM for the domain, CDP feature must be enabled and CBM
+type options need to be specified when setting CBM, and the type options (code
+and data) are mutually exclusive.
+
 =over 4
 
 =item B [I] I I
@@ -1533,6 +1540,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM when CDP is enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM when CDP is enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index c32e25c..e23a184 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -127,6 +127,59 @@ Per domain CBM settings can be shown by:
 
 `xl psr-cat-show`
 
+## Code and Data Prioritization (CDP)
+
+Code and Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cache capacity to the characteristics of the workload. CDP extends
+Cache Allocation Technology (CAT) by providing separate code and data masks
+per Class of Service (COS).
+
+CDP can be enabled by adding `psr=cdp` to Xen command line.
+
+When CDP is enabled,
+
+ * the CAT masks are re-mapped into interleaved pairs of masks for data or code
+   fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows the OS or Hypervisor to partition cache allocation in a more
+fine-grained. Code cache and data cache can be specified independently. With 
CDP
+enabled, one COS corresponds to two CBMs (code CBM & data CBM), since the sum 
of
+CBMs is fixed, that means the number of available COSes will reduce by half 
when
+CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
+
+The xl interfaces are the same with that of CAT. The difference is that CBM 
type
+can be passed as option to set code CBM or data CBM.
+
+When CDP is enabled, `-c` or `--code` option is available to set code CBM for
+the domain.
+
+When CDP is enabled, `-d` or `--data` option is available to set data CBM for
+the domain.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same co

[Xen-devel] [PATCH v6 1/3] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

Add boot parameter `psr=cdp` to enable CDP at boot time.
Intel Code/Data Prioritization (CDP) feature is based on CAT. Note that
cos_max would be half when CDP is on. struct psr_cat_cbm is extended to
support CDP operation. Extend psr_get_cat_l3_info sysctl to get CDP
status.

Signed-off-by: He Chen <he.c...@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.coop...@citrix.com>
---
Changes in v6:
* remove unnecessary parameter in cdp_is_enabled

Changes in v5:
* remove unnecessary u in psr_cat_cbm structure
* revert write_l3_cbm and put the modification to next patch
* remove duplicate PSR_CAT_FLAG_L3_CDP
---
 docs/misc/xen-command-line.markdown | 11 +++--
 xen/arch/x86/psr.c  | 48 +
 xen/arch/x86/sysctl.c   |  5 ++--
 xen/include/asm-x86/msr-index.h |  3 +++
 xen/include/asm-x86/psr.h   |  8 ++-
 xen/include/public/sysctl.h |  4 +++-
 6 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index a565c1b..416e559 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1174,9 +1174,9 @@ This option can be specified more than once (up to 8 
times at present).
 > `= `
 
 ### psr (Intel)
-> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: )`
+> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: | cdp: )`
 
-> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255`
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
 
 Platform Shared Resource(PSR) Services.  Intel Haswell and later server
 platforms offer information about the sharing of resources.
@@ -1206,6 +1206,13 @@ The following resources are available:
   the cache allocation.
   * `cat` instructs Xen to enable/disable Cache Allocation Technology.
   * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+  regarding the code cache and the data cache allocation. CDP is based on CAT.
+  * `cdp` instructs Xen to enable/disable Code and Data Prioritization. Note
+that `cos_max` of CDP is a little different from `cos_max` of CAT. With
+CDP, one COS will corespond two CBMs other than one with CAT, due to the
+sum of CBMs is fixed, that means actual `cos_max` in use will automatically
+reduce to half when CDP is enabled.
 
 ### reboot
 > `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | 
 > [c]old]`
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..e466a7e 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -21,9 +21,16 @@
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
+#define PSR_CDP(1<<2)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+};
+};
 unsigned int ref;
 };
 
@@ -43,6 +50,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_enable;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -94,6 +102,7 @@ static void __init parse_psr_param(char *s)
 
 parse_psr_bool(s, val_str, "cmt", PSR_CMT);
 parse_psr_bool(s, val_str, "cat", PSR_CAT);
+parse_psr_bool(s, val_str, "cdp", PSR_CDP);
 
 if ( val_str && !strcmp(s, "rmid_max") )
 opt_rmid_max = simple_strtoul(val_str, NULL, 0);
@@ -261,8 +270,13 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 return cat_socket_info + socket;
 }
 
+static inline bool_t cdp_is_enabled(unsigned int socket)
+{
+return cdp_socket_enable && test_bit(socket, cdp_socket_enable);
+}
+
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint32_t *flags)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +286,10 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
 
+*flags = 0;
+if ( cdp_is_enabled(socket) )
+*flags |= XEN_SYSCTL_PSR_CAT_L3_CDP;
+
 return 0;
 }
 
@@ -470,6 +488,7 @@ static void cat_cpu_init(void)
 struct psr_cat_socket_info *info;
 unsigned int socket;
 unsigned int cpu = smp_processor_id();
+uint64_t val;
 const struct cpuinfo_x86 *c = cpu_data + cpu;
 
 if ( !cpu_has(c, X86_FEATURE_CAT) || c->cpuid_level < PSR_CPUID_LEVEL_CAT )
@@ -495,8 +514,27 @@ static void cat_cpu_init(void)
 spin_lock_init(>cbm_lock);
 
 set_bit(socket, cat_socket_enable);

[Xen-devel] [PATCH v5 1/3] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

Add boot parameter `psr=cdp` to enable CDP at boot time.
Intel Code/Data Prioritization (CDP) feature is based on CAT. Note that
cos_max would be half when CDP is on. struct psr_cat_cbm is extended to
support CDP operation. Extend psr_get_cat_l3_info sysctl to get CDP
status.

Signed-off-by: He Chen <he.c...@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.coop...@citrix.com>
---
Changes in v5
* remove unnecessary u in psr_cat_cbm structure
* revert write_l3_cbm and put the modification to next patch
* remove duplicate PSR_CAT_FLAG_L3_CDP
---
 docs/misc/xen-command-line.markdown | 11 +++--
 xen/arch/x86/psr.c  | 49 ++---
 xen/arch/x86/sysctl.c   |  5 ++--
 xen/include/asm-x86/msr-index.h |  3 +++
 xen/include/asm-x86/psr.h   |  8 +-
 xen/include/public/sysctl.h |  4 ++-
 6 files changed, 70 insertions(+), 10 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index a2e427c..d92e323 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1165,9 +1165,9 @@ This option can be specified more than once (up to 8 
times at present).
 > `= `
 
 ### psr (Intel)
-> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: )`
+> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: | cdp: )`
 
-> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255`
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
 
 Platform Shared Resource(PSR) Services.  Intel Haswell and later server
 platforms offer information about the sharing of resources.
@@ -1197,6 +1197,13 @@ The following resources are available:
   the cache allocation.
   * `cat` instructs Xen to enable/disable Cache Allocation Technology.
   * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+  regarding the code cache and the data cache allocation. CDP is based on CAT.
+  * `cdp` instructs Xen to enable/disable Code and Data Prioritization. Note
+that `cos_max` of CDP is a little different from `cos_max` of CAT. With
+CDP, one COS will corespond two CBMs other than one with CAT, due to the
+sum of CBMs is fixed, that means actual `cos_max` in use will automatically
+reduce to half when CDP is enabled.
 
 ### reboot
 > `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | 
 > [c]old]`
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..37e77d1 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -21,9 +21,16 @@
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
+#define PSR_CDP(1<<2)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+};
+};
 unsigned int ref;
 };
 
@@ -43,6 +50,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_enable;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -94,6 +102,7 @@ static void __init parse_psr_param(char *s)
 
 parse_psr_bool(s, val_str, "cmt", PSR_CMT);
 parse_psr_bool(s, val_str, "cat", PSR_CAT);
+parse_psr_bool(s, val_str, "cdp", PSR_CDP);
 
 if ( val_str && !strcmp(s, "rmid_max") )
 opt_rmid_max = simple_strtoul(val_str, NULL, 0);
@@ -261,8 +270,14 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 return cat_socket_info + socket;
 }
 
+static inline bool_t cdp_is_enabled(unsigned int socket,
+unsigned long *cdp_socket_enable)
+{
+return cdp_socket_enable && test_bit(socket, cdp_socket_enable);
+}
+
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint32_t *flags)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +287,10 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
 
+*flags = 0;
+if ( cdp_is_enabled(socket, cdp_socket_enable) )
+*flags |= XEN_SYSCTL_PSR_CAT_L3_CDP;
+
 return 0;
 }
 
@@ -470,6 +489,7 @@ static void cat_cpu_init(void)
 struct psr_cat_socket_info *info;
 unsigned int socket;
 unsigned int cpu = smp_processor_id();
+uint64_t val;
 const struct cpuinfo_x86 *c = cpu_data + cpu;
 
 if ( !cpu_has(c, X86_FEATURE_CAT) || c->cpuid_level < PSR_CPUID_LEVEL_CAT )
@@ -495,8 +515,27 @@ static void cat_cpu_init(void)
 spin_lock_init(>cbm_lock);
 

[Xen-devel] [PATCH v5 2/3] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

CDP extends CAT and provides the capacity to control L3 code & data
cache. With CDP, one COS corresponds to two CMBs(code & data). cbm_type
is added to distinguish different CBM operations. Besides, new domctl
cmds are introdunced to support set/get CDP CBM. Some CAT functions to
operation CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
Reviewed-by: Andrew Cooper <andrew.coop...@citrix.com>
---
Changes in v5:
* replace -EINVAL with -ENXIO when setting code/data CBM on CDP
  disabled.
---
 xen/arch/x86/domctl.c   |  32 +++-
 xen/arch/x86/psr.c  | 188 ++--
 xen/include/asm-x86/psr.h   |  12 ++-
 xen/include/public/domctl.h |   4 +
 4 files changed, 191 insertions(+), 45 deletions(-)

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index bf62a88..734fddb 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1167,12 +1167,40 @@ long arch_do_domctl(
 {
 case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
 ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
- domctl->u.psr_cat_op.data);
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 break;
 
 case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
 ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
- >u.psr_cat_op.data);
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 copyback = 1;
 break;
 
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 37e77d1..8bfcccb 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -294,14 +294,40 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 return 0;
 }
 
-int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+int psr_get_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t *cbm, enum cbm_type type)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
+bool_t cdp_enabled = cdp_is_enabled(socket, cdp_socket_enable);
 
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+switch ( type )
+{
+case PSR_CBM_TYPE_L3:
+if ( type == PSR_CBM_TYPE_L3 && cdp_enabled )
+return -EXDEV;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+break;
+
+case PSR_CBM_TYPE_L3_CODE:
+if ( !cdp_enabled )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].code;
+break;
+
+case PSR_CBM_TYPE_L3_DATA:
+if ( !cdp_enabled )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].data;
+break;
+
+default:
+ASSERT_UNREACHABLE();
+}
 
 return 0;
 }
@@ -332,19 +358,34 @@ static bool_t psr_check_cbm(unsigned int cbm_len, 
uint64_t cbm)
 struct cos_cbm_info
 {
 unsigned int cos;
-uint64_t cbm;
+uint64_t cbm_code;
+uint64_t cbm_data;
+bool_t cdp;
 };
 
 static void do_write_l3_cbm(void *data)
 {
 struct cos_cbm_info *info = data;
 
-wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
+if ( info->cdp )
+{
+wrmsrl(MSR_IA32_PSR_L3_MASK_CODE(info->cos), info->cbm_code);
+wrmsrl(MSR_IA32_PSR_L3_MASK_DATA(info->cos), info->cbm_data);
+}
+else
+wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm_code);
 }
 
-static int write_l3_cbm(unsigned int socket, unsigned int cos, uint64_t cbm)
+static int write_l3

[Xen-devel] [PATCH v5 0/3] detect and initialize CDP (Code/Data Prioritization) feature

[He Chen]

Changes in v5:
- x86: address Andrew's and Jan's coments.
- tools: refine options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
- merge tools and docs patches.
- code style

Changes in v4:
- x86:
  * remove union member name in struct `psr_cat_cbm` (suggested by Jan)
  * fix log info of CAT & CDP (suggested by Chao & Jan)
  * add a helper `cdp_is_enabled` to tell the status of CDP and CDP initialize
failed is considered (Jan's comment)
  * XEN_SYSCTL_INTERFACE_VERSION 0x000C -> 0x000D (suggested by Jan)
  * refine CBM type check logic in get/set CBM function (suggested by Jan)
  * loop optimization in function `find_cos` (suggested by Jan)
- tools: address Chao's comments.
- docs: address Chao's comments.
- code style

Changes in v3:
- x86: remove redundant CDP field in cat_socket_enable (suggested by Chao)
- tools: simplify CBM setting function in tools (suggested by Jan)
- docs: add boot parameter description (suggested by Chao & Ian)
- code style

Changes in v2:
- x86: enable CDP by boot parameter instead of enabling/disabling CDP at
   runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This v5 patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks.

He Chen (3):
  x86: Support enable CDP by boot parameter and add get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools & docs: add tools and docs support for Intel CDP

 docs/man/xl.pod.1   |  15 +++
 docs/misc/xen-command-line.markdown |  11 +-
 docs/misc/xl-psr.markdown   |  47 ++-
 tools/libxc/include/xenctrl.h   |   7 +-
 tools/libxc/xc_psr.c|  17 ++-
 tools/libxl/libxl.h |   7 ++
 tools/libxl/libxl_psr.c |   5 +-
 tools/libxl/libxl_types.idl |   3 +
 tools/libxl/xl_cmdimpl.c|  56 +++--
 tools/libxl/xl_cmdtable.c   |   2 +
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 237 +---
 xen/arch/x86/sysctl.c   |   5 +-
 xen/include/asm-x86/msr-index.h |   3 +
 xen/include/asm-x86/psr.h   |  20 ++-
 xen/include/public/domctl.h |   4 +
 xen/include/public/sysctl.h |   4 +-
 17 files changed, 398 insertions(+), 77 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v5 3/3] tools & docs: add tools and docs support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.
Add new CDP options with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
Changes in v5:
* merge tools and docs patches
* replace EINVAL with ENXIO in libxl__psr_cat_log_err_msg
* revise options parsing in psr-cat-cbm-set and invalidate passing -c
  and -d at the same time
* refine CDP status output codes in psr_cat_hwinfo
* adjust CBM output format in command xl psr-cat-show
* docs revision

Example of new output format for command xl psr-cat-show:

CAT-only:

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME   CBM
0 Domain-0   0xf
1   centos.hvm   0xf

CDP enabled:

Socket ID   : 0
L3 Cache: 56320KB
Default CBM : 0xf
   ID NAME   CBM
0 Domain-0 code: 0xf data: 0xf
1   centos.hvm code: 0xf data: 0xf
---
 docs/man/xl.pod.1 | 15 
 docs/misc/xl-psr.markdown | 47 +++-
 tools/libxc/include/xenctrl.h |  7 --
 tools/libxc/xc_psr.c  | 17 -
 tools/libxl/libxl.h   |  7 ++
 tools/libxl/libxl_psr.c   |  5 +++-
 tools/libxl/libxl_types.idl   |  3 +++
 tools/libxl/xl_cmdimpl.c  | 56 +--
 tools/libxl/xl_cmdtable.c |  2 ++
 9 files changed, 137 insertions(+), 22 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f22c3f3..6388351 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1530,6 +1530,13 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocations, which support specifying code or data cache for
+applications. CDP is used on a per VM basis in the Xen implementation. To
+specify code or data CBM for the domain, CDP feature must be enabled and CBM
+type options need to be specified when setting CBM, and the type options (code
+and data) are mutually exclusive.
+
 =over 4
 
 =item B [I] I I
@@ -1545,6 +1552,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM when CDP is enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM when CDP is enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index 3545912..0527211 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -14,7 +14,7 @@ tracks cache utilization of memory accesses according to the 
RMID and reports
 monitored data via a counter register.
 
 For more detailed information please refer to Intel SDM chapter
-"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+"Platform Shared Resource Monitoring: Cache Monitoring Technology".
 
 In Xen's implementation, each domain in the system can be assigned a RMID
 independently, while RMID=0 is reserved for monitoring domains that don't
@@ -52,7 +52,7 @@ event type to monitor system total/local memory bandwidth. 
The same RMID can
 be used to monitor both cache usage and memory bandwidth at the same time.
 
 For more detailed information please refer to Intel SDM chapter
-"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+"Platform Shared Resource Monitoring: Cache Monitoring Technology".
 
 In Xen's implementation, MBM shares the same set of underlying monitoring
 service with CMT and can be used to monitor memory bandwidth on a per domain
@@ -91,17 +91,42 @@ For example, assuming a system with 8 portions and 3 
domains:
first domain exclusive access to half the cache, and the other two exclusive
access to one quarter each.
 
-For more detailed information please refer to Intel SDM chapter
-"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
-
 In Xen's implementation, CBM can be configured with libxl/xl interfaces but
 COS is maintained in hypervisor only. The cache partition granularity is per
 domain, each domain has COS=0 assigned by default, the corresponding CBM is
 all-ones, which means all the cache resource can be used by default.
 
+Code/Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cach

Re: [Xen-devel] [PATCH v4 3/4] tools: add tools support for Intel CDP

[He Chen]

On Thu, Sep 24, 2015 at 12:07:27PM +0100, Ian Campbell wrote:
> On Thu, 2015-09-17 at 17:35 +0800, He Chen wrote:
> > @@ -8410,20 +8415,29 @@ static void psr_cat_print_one_domain_cbm(uint32_t
> > domid, uint32_t socketid)
> >  printf("%5d%25s", domid, domain_name);
> >  free(domain_name);
> >  
> > -if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CBM,
> > -   socketid, ))
> > - printf("%#16"PRIx64, cbm);
> > -
> > +if (!cdp_enabled) {
> > +if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CBM,
> > +   socketid, ))
> > +printf("%#16"PRIx64, cbm);
> > +} else {
> > +if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_CODE,
> > +   socketid, ))
> > +printf("%10s%#8"PRIx64, "code:", cbm);
> > +if (!libxl_psr_cat_get_cbm(ctx, domid, LIBXL_PSR_CBM_TYPE_L3_DATA,
> > +   socketid, ))
> > +printf("%10s%#8"PRIx64, "data:", cbm);
> > +}
> 
> Does cdp being enabled mean that the original L3_CBM functionality is no
> longer available then?
> 
> Please could you give an example of the new output format for this command
> in the commit message.
> 

For the get side, the answer is Yes. But for the set side, L3_CBM means that
setting the same code CBM and data CBM when CDP is enabled.

> >  static int psr_cat_show(uint32_t domid)
> > @@ -8489,6 +8503,8 @@ int main_psr_cat_cbm_set(int argc, char **argv)
> >  libxl_psr_cbm_type type = LIBXL_PSR_CBM_TYPE_L3_CBM;
> >  uint64_t cbm;
> >  int ret, opt = 0;
> > +int opt_data = 0;
> > +int opt_code = 0;
> >  libxl_bitmap target_map;
> >  char *value;
> >  libxl_string_list socket_list;
> > 
> > [...]
> 
> > @@ -8517,8 +8535,19 @@ int main_psr_cat_cbm_set(int argc, char **argv)
> >  libxl_string_list_dispose(_list);
> >  free(value);
> >  break;
> > +case 'd':
> > +type = LIBXL_PSR_CBM_TYPE_L3_DATA;
> > +opt_data = 1;
> > +break;
> > +case 'c':
> > +type = LIBXL_PSR_CBM_TYPE_L3_CODE;
> > +opt_code = 1;
> > +break;
> >  }
> >  
> > +if (opt_data && opt_code)
> 
> Do you not mean !opt_data && !opt_code?
> 
> But also, isn't this assignment unnecessary since type is initialised to
> the same value when it is declared?
> 
> In fact, because of that initialisation, aren't opt_data and opt_code
> unnecessary, since you set type appropriately elsewhere.
> 
> Are -d and -c mutually exclusive, or is it expected that both can be given?
> 

-d and -c can be both given.

The initialisation of type is L3_CBM, which corresponds to the situation
that neither -d nor -c is given.

I add `if (opt_data && opt_code)` to address the situation that -d and -c
are both given.
If user gives both -d and -c options, image that without if() statement,
-d will be overwritten by the latter -c in switch, and type will be
L3_CODE instead of L3_CBM (means set both and what user wants).

I hope I had made myself clear, or is there something wrong with my
understanding?

Thanks.

> 
> ___
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v4 3/4] tools: add tools support for Intel CDP

[He Chen]

On Thu, Sep 24, 2015 at 12:22:47PM +0100, Ian Campbell wrote:
> On Thu, 2015-09-24 at 12:07 +0100, Ian Campbell wrote:
> > @@ -8517,8 +8535,19 @@ int main_psr_cat_cbm_set(int argc, char **argv)
> > >  libxl_string_list_dispose(_list);
> > >  free(value);
> > >  break;
> > > +case 'd':
> > > +type = LIBXL_PSR_CBM_TYPE_L3_DATA;
> > > +opt_data = 1;
> > > +break;
> > > +case 'c':
> > > +type = LIBXL_PSR_CBM_TYPE_L3_CODE;
> > > +opt_code = 1;
> > > +break;
> > >  }
> > >  
> > > +if (opt_data && opt_code)
> > 
> > Do you not mean !opt_data && !opt_code?
> > 
> > But also, isn't this assignment unnecessary since type is initialised to
> > the same value when it is declared?
> > 
> > In fact, because of that initialisation, aren't opt_data and opt_code
> > unnecessary, since you set type appropriately elsewhere.
> > 
> > Are -d and -c mutually exclusive, or is it expected that both can be
> > given?
> 
> Also, is there error checking for passing -c or -d when CDP is not enabled
> somewhere else?
> 
> Ian.

Yes, there is error checking in hypervisor.
hypervisor would reture an error code if -c or -d is given when CDP is
not enable.

In fact, the reture error code would be catched by
`libxl__psr_cat_log_err_msg`, and that is exactly what you and Jan
discussed in Patch 3.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v4 4/4] docs: add document to introduce CDP command

[He Chen]

On Thu, Sep 24, 2015 at 12:22:02PM +0100, Ian Campbell wrote:
> On Thu, 2015-09-17 at 17:35 +0800, He Chen wrote:
> > Add new CDP options with CAT commands in xl interface man page.
> > Add description of CDP in xl-psr.markdown.
> 
> It would have been fine to include this in the previous patch by the way.
> 

You mean include both xl man page and xl-psr.markdown in the previous
patch? Or just only xl man page?

> > +code or data CBM for the domain, CDP feature must be enabled and CBM type
> > +options need to be specified when setting CBM.
> 
> I asked on patch 3 whether these options were mutually exclusive or not,
> the answer should be reflected in the documentation too please.
> 

Agreed.
Both code and data options can be specified at the same time.
I will make it clear in the documentation as possible as I can.

> > +
> > +When CDP is enabled,
> > +
> > + * the CAT masks are re-mapped into interleaved pairs of masks for data or 
> > code
> > +   fetches.
> > +
> > + * the range of COS for CAT is re-indexed, with the lower-half of the COS
> > +   range available for CDP.
> > +
> > +CDP allows OS or Hypervisor to partition cache allocation more 
> > fine-grained,
> 
>  ^the   ^in a  
> 
> > +code cache and data cache can be specified respectively. With CDP enabled,
> 
> End fine-grained with a full-stop and then:
> 
> The code cache and data cache can be specified separately
> 
> or s/separately/independently/?
> 
> (I think, I'm not 100% sure what you meant by "respectively", so maybe the
> suggestion is wrong)
> 

Maybe I make words a little confused here.
With CDP enabled, user can specify code cache (or data cache) only, and
the other would remain previous value (if no previous value, it would
keep as default value).
So, I think independently would be fine here.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v4 3/4] tools: add tools support for Intel CDP

[He Chen]

> Quoting the relevant bits of code for clarity:
>  libxl_psr_cbm_type type = LIBXL_PSR_CBM_TYPE_L3_CBM;
> ...
> case 'd':
> type = LIBXL_PSR_CBM_TYPE_L3_DATA;
> opt_data = 1;
> break;
> case 'c':
> type = LIBXL_PSR_CBM_TYPE_L3_CODE;
> opt_code = 1;
> break;
>  }
>  
> if (opt_data && opt_code)
> type = LIBXL_PSR_CBM_TYPE_L3_CBM;
> 
> So the behaviour if -d and -c are given is exactly the same as if neither
> of them were given, i.e. type = LIBXL_PSR_CBM_TYPE_L3_CBM? Is that correct
> and intended?

Yes.

> If so then I think it would be clearer to only set opt_* during option
> parsing and then to figure out the correct LIBXL_PSR_CBM_TYPE_* explicitly
> afterwards, rather than have the code cycle through data->code->cbm.
> 
> Or just outlaw passing both -d and -c together since it is confusing and
> equivalent to passing neither anyway.

Yes, as you said, if user just passes one option -d (or -c), things would
be done during option parsing, there is no need to add the if().

But the key point is that I am not sure how to address outlaw passing both
-d and -c together (is it allowed?). If it is permitted, the behaviour is
the same as passing neither indeed, and the if() is needed to avoid latter
option overwritting former option.

What's your suggestion? Sorry, I am a little confused.
Omit former opiton when both options are given and remove if()?
Or something else?

Thanks for your time.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v4 4/4] docs: add document to introduce CDP command

[He Chen]

On Fri, Sep 25, 2015 at 10:58:58AM +0100, Ian Campbell wrote:
> On Fri, 2015-09-25 at 17:29 +0800, He Chen wrote:
> > On Thu, Sep 24, 2015 at 12:22:02PM +0100, Ian Campbell wrote:
> > > On Thu, 2015-09-17 at 17:35 +0800, He Chen wrote:
> > > > Add new CDP options with CAT commands in xl interface man page.
> > > > Add description of CDP in xl-psr.markdown.
> > > 
> > > It would have been fine to include this in the previous patch by the
> > > way.
> > > 
> > 
> > You mean include both xl man page and xl-psr.markdown in the previous
> > patch? Or just only xl man page?
> 
> All of the docs could validly be included with the patch which introduces
> the feature. i.e. you could fold patch #4 into #3 completely.
> 
> You don't have to if you don't want to.
> 

Thanks for your reminding, I will merge patch #3 and #4 in next version.

> > Maybe I make words a little confused here.
> > With CDP enabled, user can specify code cache (or data cache) only, and
> > the other would remain previous value (if no previous value, it would
> > keep as default value).
> 
> The value is just enabled or disabled, so do I understand correctly that
> this is what you can do if cdp is enabled:
> 
> [now: code cbm=; data cbm=]
> 
> xl psr-cat-cbm-set -c  0xdeadbeed
> [now: code cbm=0xdeadbeef; data cbm=]
> 
> xl psr-cat-cbm-set -d  0xf00fb00f
> [now: code cbm=0xdeadbeef; data cbm=0xf00fb00f]
> 
> xl psr-cat-cbm-set -c -d  0xd00dfeed
> 
> [now: code cbm=0xd00dfeed; data cbm=0xd00dfeed]
> 
> xl psr-cat-cbm-set  0xee11ee11
> 
> [now: code cbm=0xee11ee11; data cbm=0xee11ee11]
> 
> Is that right?

Yes.

> 
> And if cdp is not enabled:
> 
> [now: cbm=]
> 
> xl psr-cat-cbm-set -c  0xdeadbeed
> *ERRROR*
> [now: cbm=]
> 
> xl psr-cat-cbm-set -d  0xf00fb00f
> *ERRROR*
> [now: cbm=]
> 

Right above.

> xl psr-cat-cbm-set -c -d  0xd00dfeed
> 
> *ERRROR*
> [now: cbm=]
> 

In current code, it is valid since -c & -d have the same behaviour as
neither of them.
So, it will not show error, and cbm=0xd00dfeed

What do you think of this? Is it proper to do so?

> xl psr-cat-cbm-set  0xee11ee11
> 
> [now: cbm=0xee11ee11]
> 

Right.

> > So, I think independently would be fine here.
> 
> If I'm correct in all the above then yes.
> 
> Ian.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v4 3/4] tools: add tools support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 tools/libxc/include/xenctrl.h |  7 +--
 tools/libxc/xc_psr.c  | 17 ++-
 tools/libxl/libxl.h   |  7 +++
 tools/libxl/libxl_psr.c   |  5 -
 tools/libxl/libxl_types.idl   |  3 +++
 tools/libxl/xl_cmdimpl.c  | 49 ++-
 tools/libxl/xl_cmdtable.c |  3 +++
 7 files changed, 77 insertions(+), 14 deletions(-)

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index de3c0ad..54f2069 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2798,7 +2798,9 @@ enum xc_psr_cmt_type {
 typedef enum xc_psr_cmt_type xc_psr_cmt_type;
 
 enum xc_psr_cat_type {
-XC_PSR_CAT_L3_CBM = 1,
+XC_PSR_CAT_L3_CBM  = 1,
+XC_PSR_CAT_L3_CODE = 2,
+XC_PSR_CAT_L3_DATA = 3,
 };
 typedef enum xc_psr_cat_type xc_psr_cat_type;
 
@@ -2824,7 +2826,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, 
uint32_t domid,
xc_psr_cat_type type, uint32_t target,
uint64_t *data);
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len);
+   uint32_t *cos_max, uint32_t *cbm_len,
+   bool *cdp_enabled);
 #endif
 
 #endif /* XENCTRL_H */
diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index d8b3a51..5bbe950 100644
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -260,6 +260,12 @@ int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -287,6 +293,12 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -306,7 +318,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 }
 
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len)
+   uint32_t *cos_max, uint32_t *cbm_len,
+   bool *cdp_enabled)
 {
 int rc;
 DECLARE_SYSCTL;
@@ -320,6 +333,8 @@ int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t 
socket,
 {
 *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
 *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+*cdp_enabled = sysctl.u.psr_cat_op.u.l3_info.flags &
+   XEN_SYSCTL_PSR_CAT_L3_CDP;
 }
 
 return rc;
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5f9047c..611e98d 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -796,6 +796,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, 
libxl_mac *src);
  * If this is defined, the Cache Allocation Technology feature is supported.
  */
 #define LIBXL_HAVE_PSR_CAT 1
+
+/*
+ * LIBXL_HAVE_PSR_CDP
+ *
+ * If this is defined, the Code and Data Prioritization feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CDP 1
 #endif
 
 /*
diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 3378239..62963cf 100644
--- a/tools/libxl/libxl_psr.c
+++ b/tools/libxl/libxl_psr.c
@@ -87,6 +87,9 @@ static void libxl__psr_cat_log_err_msg(libxl__gc *gc, int err)
 case EEXIST:
 msg = "The same CBM is already set to this domain";
 break;
+case EINVAL:
+msg = "Unable to set code or data CBM when CDP is disabled";
+break;
 
 default:
 libxl__psr_log_err_msg(gc, err);
@@ -352,7 +355,7 @@ int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, 
libxl_psr_cat_info **info,
 
 for (i = 0; i < nr_sockets; i++) {
 if (xc_psr_cat_get_l3_info(ctx->xch, i, [i].cos_max,
-[i].cbm_len)) {
+   [i].cbm_len, [i].cdp_enabled)) {
 libxl__psr_cat_log_err_msg(gc, errno);
 rc = ERROR_FAIL;
 free(ptr);
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index ef346e7..fa017ad 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -787,9 +787,12 @@ libxl_psr_cmt_type = Enumeration("psr_cmt_type", [
 libxl_psr_cbm_type = E

[Xen-devel] [PATCH v4 1/4] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

Add boot parameter `psr=cdp` to enable CDP at boot time.
Intel Code/Data Prioritization(CDP) feature is based on CAT. Note that
cos_max would be half when CDP is on. struct psr_cat_cbm is extended to
support CDP operation. Extend psr_get_cat_l3_info sysctl to get CDP
status.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/misc/xen-command-line.markdown | 11 -
 xen/arch/x86/psr.c  | 84 ++---
 xen/arch/x86/sysctl.c   |  5 ++-
 xen/include/asm-x86/msr-index.h |  3 ++
 xen/include/asm-x86/psr.h   | 11 -
 xen/include/public/sysctl.h |  4 +-
 6 files changed, 98 insertions(+), 20 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index a2e427c..d92e323 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1165,9 +1165,9 @@ This option can be specified more than once (up to 8 
times at present).
 > `= `
 
 ### psr (Intel)
-> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: )`
+> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: | cdp: )`
 
-> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255`
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
 
 Platform Shared Resource(PSR) Services.  Intel Haswell and later server
 platforms offer information about the sharing of resources.
@@ -1197,6 +1197,13 @@ The following resources are available:
   the cache allocation.
   * `cat` instructs Xen to enable/disable Cache Allocation Technology.
   * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+  regarding the code cache and the data cache allocation. CDP is based on CAT.
+  * `cdp` instructs Xen to enable/disable Code and Data Prioritization. Note
+that `cos_max` of CDP is a little different from `cos_max` of CAT. With
+CDP, one COS will corespond two CBMs other than one with CAT, due to the
+sum of CBMs is fixed, that means actual `cos_max` in use will automatically
+reduce to half when CDP is enabled.
 
 ### reboot
 > `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | 
 > [c]old]`
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..e44ed8b 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -21,9 +21,16 @@
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
+#define PSR_CDP(1<<2)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+};
+} u;
 unsigned int ref;
 };
 
@@ -43,6 +50,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_enable;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -94,6 +102,7 @@ static void __init parse_psr_param(char *s)
 
 parse_psr_bool(s, val_str, "cmt", PSR_CMT);
 parse_psr_bool(s, val_str, "cat", PSR_CAT);
+parse_psr_bool(s, val_str, "cdp", PSR_CDP);
 
 if ( val_str && !strcmp(s, "rmid_max") )
 opt_rmid_max = simple_strtoul(val_str, NULL, 0);
@@ -261,8 +270,14 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 return cat_socket_info + socket;
 }
 
+static inline bool_t cdp_is_enabled(unsigned int socket,
+unsigned long *cdp_socket_enable)
+{
+return (cdp_socket_enable && test_bit(socket, cdp_socket_enable));
+}
+
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint32_t *flags)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +287,10 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
 
+*flags = 0;
+if ( cdp_is_enabled(socket, cdp_socket_enable) )
+*flags |= PSR_CAT_FLAG_L3_CDP;
+
 return 0;
 }
 
@@ -282,7 +301,7 @@ int psr_get_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t *cbm)
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
 
 return 0;
 }
@@ -313,19 +332,34 @@ static bool_t psr_check_cbm(unsigned int cbm_len, 
uint64_t cbm)
 struct cos_cbm_info
 {
 unsigned int cos;
-uint64_t cbm;
+uint64_t cbm_code;
+uint64_t cbm_data;
+bool_t cdp;
 };
 
 static void do_write_l3_cbm(void *data)
 {
 struct cos_cbm_info *info = data;
 
-wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info-

[Xen-devel] [PATCH v4 0/4] detect and initialize CDP (Code/Data Prioritization) feature

[He Chen]

Changes in v4:
- x86:
  * remove union member name in struct `psr_cat_cbm` (suggested by Jan)
  * fix log info of CAT & CDP (suggested by Chao & Jan)
  * add a helper `cdp_is_enabled` to tell the status of CDP and CDP initialize
failed is considered (Jan's comment)
  * XEN_SYSCTL_INTERFACE_VERSION 0x000C -> 0x000D (suggested by Jan)
  * refine CBM type check logic in get/set CBM function (suggested by Jan)
  * loop optimization in function `find_cos` (suggested by Jan)
- tools: Address Chao's comments.
- docs: Address Chao's comments.
- code style

Changes in v3:
- x86: remove redundant CDP field in cat_socket_enable (suggested by Chao)
- tools: simplify CBM setting function in tools (suggested by Jan)
- docs: Add boot parameter description (suggested by Chao & Ian)
- code style

Changes in v2:
- x86: Enable CDP by boot parameter instead of enabling/disabling CDP at
   runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This v4 patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks

He Chen (4):
  x86: Support enable CDP by boot parameter and add get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools: add tools support for Intel CDP
  docs: add document to introduce CDP command

 docs/man/xl.pod.1   |  14 +++
 docs/misc/xen-command-line.markdown |  11 +-
 docs/misc/xl-psr.markdown   |  44 ++-
 tools/libxc/include/xenctrl.h   |   7 +-
 tools/libxc/xc_psr.c|  17 ++-
 tools/libxl/libxl.h |   7 ++
 tools/libxl/libxl_psr.c |   5 +-
 tools/libxl/libxl_types.idl |   3 +
 tools/libxl/xl_cmdimpl.c|  49 ++--
 tools/libxl/xl_cmdtable.c   |   3 +
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 239 +---
 xen/arch/x86/sysctl.c   |   5 +-
 xen/include/asm-x86/msr-index.h |   3 +
 xen/include/asm-x86/psr.h   |  23 +++-
 xen/include/public/domctl.h |   4 +
 xen/include/public/sysctl.h |   4 +-
 17 files changed, 395 insertions(+), 75 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v4 4/4] docs: add document to introduce CDP command

[He Chen]

Add new CDP options with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/man/xl.pod.1 | 14 ++
 docs/misc/xl-psr.markdown | 44 +++-
 2 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f22c3f3..3c7107d 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1530,6 +1530,12 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocation, which support specifying code or data cache for
+applications. CDP is used on VM basis in the Xen implementation. To specify
+code or data CBM for the domain, CDP feature must be enabled and CBM type
+options need to be specified when setting CBM.
+
 =over 4
 
 =item B [I] I I
@@ -1545,6 +1551,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM when CDP is enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM when CDP is enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index 3545912..5eb97cc 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -14,7 +14,7 @@ tracks cache utilization of memory accesses according to the 
RMID and reports
 monitored data via a counter register.
 
 For more detailed information please refer to Intel SDM chapter
-"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+"17.15 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
 
 In Xen's implementation, each domain in the system can be assigned a RMID
 independently, while RMID=0 is reserved for monitoring domains that don't
@@ -91,17 +91,42 @@ For example, assuming a system with 8 portions and 3 
domains:
first domain exclusive access to half the cache, and the other two exclusive
access to one quarter each.
 
-For more detailed information please refer to Intel SDM chapter
-"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
-
 In Xen's implementation, CBM can be configured with libxl/xl interfaces but
 COS is maintained in hypervisor only. The cache partition granularity is per
 domain, each domain has COS=0 assigned by default, the corresponding CBM is
 all-ones, which means all the cache resource can be used by default.
 
+Code/Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cache capacity to the characteristics of the workload. CDP extends
+Cache Allocation Technology (CAT) by providing separate code and data masks
+per Class of Service (COS).
+
+CDP can be enabled by adding `psr=cdp` to Xen bootline.
+
+When CDP is enabled,
+
+ * the CAT masks are re-mapped into interleaved pairs of masks for data or code
+   fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows OS or Hypervisor to partition cache allocation more fine-grained,
+code cache and data cache can be specified respectively. With CDP enabled,
+one COS corresponds to two CBMs (code CBM & data CBM), since the sum of CBMs
+is fixed, that means the number of available COSes will reduce to half when
+CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"17.16 - Platform Shared Resource Control: Cache Allocation Technology".
+
 ### xl interfaces
 
-System CAT information such as maximum COS and CBM length can be obtained by:
+System CAT information such as maximum COS, CBM length and CDP status can be
+obtained by:
 
 `xl psr-hwinfo --cat`
 
@@ -116,6 +141,15 @@ A cbm is valid only when:
obtained with `xl psr-hwinfo --cat`.
  * All the set bits are contiguous.
 
+When CDP is enabled, `-c` or `--code` option is available to set code CBM for
+the domain.
+
+When CDP is enabled, `-d` or `--data` option is available to set data CBM for
+the domain.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same code CBM
+and data CBM will be set for the domain.
+
 In a multi-socket system, the same cbm will be set on each socket by default.
 Per socket cbm can be specified with the `--socket SOCKET` option.
 
-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v4 2/4] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

CDP extends CAT and provides the capacity to control L3 code & data
cache. With CDP, one COS corresponds to two CMBs(code & data). cbm_type
is added to distinguish different CBM operations. Besides, new domctl
cmds are introdunced to support set/get CDP CBM. Some CAT functions to
operation CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 165 ++--
 xen/include/asm-x86/psr.h   |  12 +++-
 xen/include/public/domctl.h |   4 ++
 4 files changed, 172 insertions(+), 41 deletions(-)

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index bf62a88..734fddb 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1167,12 +1167,40 @@ long arch_do_domctl(
 {
 case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
 ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
- domctl->u.psr_cat_op.data);
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 break;
 
 case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
 ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
- >u.psr_cat_op.data);
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 copyback = 1;
 break;
 
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index e44ed8b..c5519b7 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -294,14 +294,40 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 return 0;
 }
 
-int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+int psr_get_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t *cbm, enum cbm_type type)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
+bool_t cdp_enabled = cdp_is_enabled(socket, cdp_socket_enable);
 
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+switch ( type )
+{
+case PSR_CBM_TYPE_L3:
+if ( type == PSR_CBM_TYPE_L3 && cdp_enabled )
+return -EXDEV;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+break;
+
+case PSR_CBM_TYPE_L3_CODE:
+if ( !cdp_enabled )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.code;
+break;
+
+case PSR_CBM_TYPE_L3_DATA:
+if ( !cdp_enabled )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.data;
+break;
+
+default:
+ASSERT_UNREACHABLE();
+}
 
 return 0;
 }
@@ -375,10 +401,48 @@ static int write_l3_cbm(unsigned int socket, unsigned int 
cos,
 return 0;
 }
 
-int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
+static int find_cos(struct psr_cat_cbm *map, int cos_max,
+uint64_t cbm_code, uint64_t cbm_data, bool_t cdp_enabled)
 {
-unsigned int old_cos, cos;
-struct psr_cat_cbm *map, *found = NULL;
+unsigned int cos;
+
+for ( cos = 0; cos <= cos_max; cos++ )
+{
+if( map[cos].ref &&
+((!cdp_enabled && map[cos].u.cbm == cbm_code) ||
+(cdp_enabled && map[cos].u.code == cbm_code &&
+map[cos].u.data == cbm_data)))
+return cos;
+}
+
+return -ENOENT;
+}
+
+static int pick_avail_cos(struct psr_cat_cbm *map, unsigned int cos_max,
+  unsigned int 

[Xen-devel] [PATCH v3 0/4] Intel Code and Data Prioritization (CDP) feature enabling

[He Chen]

Changes in v3:
- x86: remove redundant CDP field in cat_socket_enable (suggested by Chao)
- tools: simplify CBM setting function in tools (suggested by Jan)
- docs: Add boot parameter description (suggested by Chao & Ian)
- code style


Changes in v2:
- x86: Enable CDP by boot parameter instead of enabling/disabling CDP at
runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks

He Chen (4):
  x86: Support enable CDP by boot parameter and add get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools: add tools support for Intel CDP
  docs: add document to introduce CDP command

 docs/man/xl.pod.1   |  14 +++
 docs/misc/xen-command-line.markdown |   8 +-
 docs/misc/xl-psr.markdown   |  42 ++-
 tools/libxc/include/xenctrl.h   |   7 +-
 tools/libxc/xc_psr.c|  17 ++-
 tools/libxl/libxl.h |   1 +
 tools/libxl/libxl_psr.c |  20 ++-
 tools/libxl/libxl_types.idl |   3 +
 tools/libxl/xl_cmdimpl.c|  49 ++--
 tools/libxl/xl_cmdtable.c   |   3 +
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 240 +---
 xen/arch/x86/sysctl.c   |   5 +-
 xen/include/asm-x86/msr-index.h |   3 +
 xen/include/asm-x86/psr.h   |  23 +++-
 xen/include/public/domctl.h |   4 +
 xen/include/public/sysctl.h |   2 +
 17 files changed, 399 insertions(+), 74 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v3 2/4] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

CDP extends CAT and provides the capacity to control L3 code & data
cache. With CDP, one COS correspond to two CMBs(code & data). cbm_type
is added to support distinguish different CBM operation. Besides, new
domctl cmds are introdunced to support set/get CDP CBM. Some CAT
functions to operation CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 166 ++--
 xen/include/asm-x86/psr.h   |  12 +++-
 xen/include/public/domctl.h |   4 ++
 4 files changed, 174 insertions(+), 40 deletions(-)

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index bf62a88..734fddb 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1167,12 +1167,40 @@ long arch_do_domctl(
 {
 case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
 ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
- domctl->u.psr_cat_op.data);
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 break;
 
 case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
 ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
- >u.psr_cat_op.data);
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 copyback = 1;
 break;
 
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index cf70f90..c55493d 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -288,14 +288,39 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 return 0;
 }
 
-int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+int psr_get_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t *cbm, enum cbm_type type)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
+bool_t cdp_enabled = test_bit(socket, cdp_socket_enable);
 
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+if ( type == PSR_CBM_TYPE_L3 && cdp_enabled )
+return -EXDEV;
+
+if ( (type == PSR_CBM_TYPE_L3_CODE || type == PSR_CBM_TYPE_L3_DATA)
+&& !cdp_enabled )
+return -EXDEV;
+
+switch ( type )
+{
+case PSR_CBM_TYPE_L3:
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+break;
+
+case PSR_CBM_TYPE_L3_CODE:
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.code;
+break;
+
+case PSR_CBM_TYPE_L3_DATA:
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.data;
+break;
+
+default:
+return -EINVAL;
+}
 
 return 0;
 }
@@ -369,10 +394,53 @@ static int write_l3_cbm(unsigned int socket, unsigned int 
cos,
 return 0;
 }
 
-int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
+static int find_cos(struct psr_cat_cbm *map, int cos_max,
+uint64_t cbm_code, uint64_t cbm_data, bool_t cdp_enabled)
 {
-unsigned int old_cos, cos;
-struct psr_cat_cbm *map, *found = NULL;
+unsigned int cos;
+
+if ( !cdp_enabled )
+{
+for ( cos = 0; cos <= cos_max; cos++ )
+if ( map[cos].ref && map[cos].u.cbm == cbm_code )
+return cos;
+}
+else
+{
+for ( cos = 0; cos <= cos_max; cos++ )
+if ( map[cos].ref && map[cos].u.cdp.code == cbm_code &&
+ map[cos].u.cdp.data == cbm_data )
+return cos;
+}
+
+return -ENOENT;
+}
+
+static int pick_avail_cos(struct psr_cat_cbm *map, int cos_max, int old_cos)
+{
+i

[Xen-devel] [PATCH v3 3/4] tools: add tools support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 tools/libxc/include/xenctrl.h |  7 +--
 tools/libxc/xc_psr.c  | 17 ++-
 tools/libxl/libxl.h   |  1 +
 tools/libxl/libxl_psr.c   | 20 +-
 tools/libxl/libxl_types.idl   |  3 +++
 tools/libxl/xl_cmdimpl.c  | 49 ++-
 tools/libxl/xl_cmdtable.c |  3 +++
 7 files changed, 86 insertions(+), 14 deletions(-)

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index de3c0ad..54f2069 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2798,7 +2798,9 @@ enum xc_psr_cmt_type {
 typedef enum xc_psr_cmt_type xc_psr_cmt_type;
 
 enum xc_psr_cat_type {
-XC_PSR_CAT_L3_CBM = 1,
+XC_PSR_CAT_L3_CBM  = 1,
+XC_PSR_CAT_L3_CODE = 2,
+XC_PSR_CAT_L3_DATA = 3,
 };
 typedef enum xc_psr_cat_type xc_psr_cat_type;
 
@@ -2824,7 +2826,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, 
uint32_t domid,
xc_psr_cat_type type, uint32_t target,
uint64_t *data);
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len);
+   uint32_t *cos_max, uint32_t *cbm_len,
+   bool *cdp_enabled);
 #endif
 
 #endif /* XENCTRL_H */
diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index d8b3a51..5bbe950 100644
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -260,6 +260,12 @@ int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -287,6 +293,12 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -306,7 +318,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 }
 
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len)
+   uint32_t *cos_max, uint32_t *cbm_len,
+   bool *cdp_enabled)
 {
 int rc;
 DECLARE_SYSCTL;
@@ -320,6 +333,8 @@ int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t 
socket,
 {
 *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
 *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+*cdp_enabled = sysctl.u.psr_cat_op.u.l3_info.flags &
+   XEN_SYSCTL_PSR_CAT_L3_CDP;
 }
 
 return rc;
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5f9047c..d824105 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -796,6 +796,7 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, 
libxl_mac *src);
  * If this is defined, the Cache Allocation Technology feature is supported.
  */
 #define LIBXL_HAVE_PSR_CAT 1
+
 #endif
 
 /*
diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 3378239..140af4d 100644
--- a/tools/libxl/libxl_psr.c
+++ b/tools/libxl/libxl_psr.c
@@ -297,6 +297,7 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
 GC_INIT(ctx);
 int rc;
 int socketid, nr_sockets;
+libxl_psr_cat_info *info;
 
 rc = libxl__count_physical_sockets(gc, _sockets);
 if (rc) {
@@ -304,6 +305,22 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
 goto out;
 }
 
+rc = libxl_psr_cat_get_l3_info(ctx, , _sockets);
+if (rc) {
+LOGE(ERROR, "Failed to get cat info");
+goto out;
+}
+
+if (!info->cdp_enabled &&
+   (type == LIBXL_PSR_CBM_TYPE_L3_CODE ||
+type == LIBXL_PSR_CBM_TYPE_L3_DATA))
+{
+LOGE(ERROR, "Unable to set Code/Data CBM with CDP disabled");
+rc = EINVAL;
+free(info);
+goto out;
+}
+
 libxl_for_each_set_bit(socketid, *target_map) {
 if (socketid >= nr_sockets)
 break;
@@ -312,6 +329,7 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
 rc = ERROR_FAIL;
 }
 }
+free(info);
 
 out:
 GC_FREE;
@@ -352,7 +370,7 @@ int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, 
libxl_psr_cat_info 

[Xen-devel] [PATCH v3 1/4] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

Add boot parameter `psr=cdp` to enable CDP at boot time.
Intel Code/Data Prioritization(CDP) feature is based on CAT. Note that
cos_max would be half when CDP is on. struct psr_cat_cbm is extended to
support CDP operation. Extend psr_get_cat_l3_info sysctl to get CDP
status.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/misc/xen-command-line.markdown |  8 +++-
 xen/arch/x86/psr.c  | 84 ++---
 xen/arch/x86/sysctl.c   |  5 ++-
 xen/include/asm-x86/msr-index.h |  3 ++
 xen/include/asm-x86/psr.h   | 11 -
 xen/include/public/sysctl.h |  2 +
 6 files changed, 93 insertions(+), 20 deletions(-)

diff --git a/docs/misc/xen-command-line.markdown 
b/docs/misc/xen-command-line.markdown
index a2e427c..8ca1d7f 100644
--- a/docs/misc/xen-command-line.markdown
+++ b/docs/misc/xen-command-line.markdown
@@ -1165,9 +1165,9 @@ This option can be specified more than once (up to 8 
times at present).
 > `= `
 
 ### psr (Intel)
-> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: )`
+> `= List of ( cmt: | rmid_max: | cat: | 
cos_max: | cdp: )`
 
-> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255`
+> Default: `psr=cmt:0,rmid_max:255,cat:0,cos_max:255,cdp:0`
 
 Platform Shared Resource(PSR) Services.  Intel Haswell and later server
 platforms offer information about the sharing of resources.
@@ -1197,6 +1197,10 @@ The following resources are available:
   the cache allocation.
   * `cat` instructs Xen to enable/disable Cache Allocation Technology.
   * `cos_max` indicates the max value for COS ID.
+* Code and Data Prioritization Technology (Broadwell and later). Information
+  regarding the code cache and the data cache allocation. CDP is based on CAT.
+  * `cdp` instructs Xen to enable/disable Code and Data Prioritization.
+
 
 ### reboot
 > `= t[riple] | k[bd] | a[cpi] | p[ci] | P[ower] | e[fi] | n[o] [, [w]arm | 
 > [c]old]`
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..cf70f90 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -21,9 +21,16 @@
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
+#define PSR_CDP(1<<2)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+} cdp;
+} u;
 unsigned int ref;
 };
 
@@ -43,6 +50,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_enable;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -94,6 +102,7 @@ static void __init parse_psr_param(char *s)
 
 parse_psr_bool(s, val_str, "cmt", PSR_CMT);
 parse_psr_bool(s, val_str, "cat", PSR_CAT);
+parse_psr_bool(s, val_str, "cdp", PSR_CDP);
 
 if ( val_str && !strcmp(s, "rmid_max") )
 opt_rmid_max = simple_strtoul(val_str, NULL, 0);
@@ -262,7 +271,7 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 }
 
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint32_t *flags)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +281,10 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
 
+*flags = 0;
+if ( test_bit(socket, cdp_socket_enable) )
+*flags |= PSR_CAT_FLAG_L3_CDP;
+
 return 0;
 }
 
@@ -282,7 +295,7 @@ int psr_get_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t *cbm)
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
 
 return 0;
 }
@@ -313,19 +326,34 @@ static bool_t psr_check_cbm(unsigned int cbm_len, 
uint64_t cbm)
 struct cos_cbm_info
 {
 unsigned int cos;
-uint64_t cbm;
+uint64_t cbm_code;
+uint64_t cbm_data;
+bool_t cdp;
 };
 
 static void do_write_l3_cbm(void *data)
 {
 struct cos_cbm_info *info = data;
 
-wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
+if ( info->cdp )
+{
+wrmsrl(MSR_IA32_PSR_L3_MASK_CODE(info->cos), info->cbm_code);
+wrmsrl(MSR_IA32_PSR_L3_MASK_DATA(info->cos), info->cbm_data);
+}
+else
+wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm_code);
 }
 
-static int write_l3_cbm(unsigned int socket, unsigned int cos, uint64_t cbm)
+static int write_l3_cbm(unsigned int socket, unsigned int cos,
+uint64_t cbm_code, uint64_t cbm_data, bool_t cdp)
 {
-struct cos_cbm_info 

[Xen-devel] [PATCH v3 4/4] docs: add document to introduce CDP command

[He Chen]

Add new CDP option with CAT commands in xl interface man page.
Add description of CDP in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/man/xl.pod.1 | 14 ++
 docs/misc/xl-psr.markdown | 42 +-
 2 files changed, 51 insertions(+), 5 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f22c3f3..3801039 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1530,6 +1530,12 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocation, which support specify code or data cache for
+applications. CDP is used on VM basis in the Xen implementation. To specify
+code or data CBM for the domain, CDP feature must be enabled and CBM type
+options need to be specified when setting CBM.
+
 =over 4
 
 =item B [I] I I
@@ -1545,6 +1551,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM with CDP enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM with CDP enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index 3545912..538ea60 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -14,7 +14,7 @@ tracks cache utilization of memory accesses according to the 
RMID and reports
 monitored data via a counter register.
 
 For more detailed information please refer to Intel SDM chapter
-"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+"17.15 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
 
 In Xen's implementation, each domain in the system can be assigned a RMID
 independently, while RMID=0 is reserved for monitoring domains that don't
@@ -91,17 +91,42 @@ For example, assuming a system with 8 portions and 3 
domains:
first domain exclusive access to half the cache, and the other two exclusive
access to one quarter each.
 
-For more detailed information please refer to Intel SDM chapter
-"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
-
 In Xen's implementation, CBM can be configured with libxl/xl interfaces but
 COS is maintained in hypervisor only. The cache partition granularity is per
 domain, each domain has COS=0 assigned by default, the corresponding CBM is
 all-ones, which means all the cache resource can be used by default.
 
+Code/Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cache capacity to the characteristics of the workload. CDP extends
+Cache Allocation Technology (CAT) by providing separate code and data masks
+per Class of Service (COS).
+
+CDP is disabled on the processor by default. If the CAT MSRs are used without
+enabling CDP, the processor operates in a traditional CAT-only mode.
+
+When CDP is enabled,
+
+ * the CAT mask MSRs are re-mapped into interleaved pairs of mask MSRs for
+   data or code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows OS or Hypervisor to partition cache allocation more fine-grained,
+code cache and data cache can be specified respectively. With CDP enabled,
+one COS corresponds to two CBMs (code CBM & data CBM), which means the number
+of available COS will reduce to half when CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"17.16 - Platform Shared Resource Control: Cache Allocation Technology".
+
 ### xl interfaces
 
-System CAT information such as maximum COS and CBM length can be obtained by:
+System CAT information such as maximum COS, CBM length and CDP status can be
+obtained by:
 
 `xl psr-hwinfo --cat`
 
@@ -119,6 +144,13 @@ A cbm is valid only when:
 In a multi-socket system, the same cbm will be set on each socket by default.
 Per socket cbm can be specified with the `--socket SOCKET` option.
 
+To specify code CBM for the domain, `-c` or `--code` option is needed.
+
+To specify data CBM for the domain, `-d` or `--data` option is needed.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same code CBM
+and data CBM will be set for the domain.
+
 Setting the CBM may not be successful if insufficient COS is available. In
 such case unused COS(es) may be freed by setting CBM of all related domains to
 its default value(all-ones).
-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH v2 1/4] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

On Wed, Sep 09, 2015 at 03:04:35PM +0800, Chao Peng wrote:
> On Wed, Sep 09, 2015 at 01:16:45PM +0800, He Chen wrote:
> > Intel Code/Data Prioritization(CDP) feature is based on CAT. cdp_enabled
> > is added to CAT socket info to indicate CDP is on or off on the socket,
> > note that cos_max would be half when CDP is on. struct psr_cat_cbm is
> > extended to support CDP operation. Extend psr_get_cat_l3_info sysctl to
> > get CDP status.
> > 
> > Signed-off-by: He Chen <he.c...@linux.intel.com>
> > ---
> >  xen/arch/x86/psr.c  | 84 
> > ++---
> >  xen/arch/x86/sysctl.c   |  5 ++-
> >  xen/include/asm-x86/msr-index.h |  3 ++
> >  xen/include/asm-x86/psr.h   | 11 +-
> >  xen/include/public/sysctl.h |  2 +
> >  5 files changed, 89 insertions(+), 16 deletions(-)
> > 
> > diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
> > index c0daa2e..ba0f726 100644
> > --- a/xen/arch/x86/psr.c
> > +++ b/xen/arch/x86/psr.c
> > @@ -17,13 +17,21 @@
> >  #include 
> >  #include 
> >  #include 
> > +#include 
> 
> Is this still needed?
> 

Obviously not, my mistake.

> >  #include 
> >  
> >  #define PSR_CMT(1<<0)
> >  #define PSR_CAT(1<<1)
> > +#define PSR_CDP(1<<2)
> >  
> >  struct psr_cat_cbm {
> > -uint64_t cbm;
> > +union {
> > +uint64_t cbm;
> > +struct {
> > +uint64_t code;
> > +uint64_t data;
> > +} cdp;
> > +} u;
> >  unsigned int ref;
> >  };
> >  
> > @@ -32,6 +40,7 @@ struct psr_cat_socket_info {
> >  unsigned int cos_max;
> >  struct psr_cat_cbm *cos_to_cbm;
> >  spinlock_t cbm_lock;
> 
> As you defined the cdp stauts for each socket here ...
> > +bool_t cdp_enabled;
> >  };
> >  
> >  struct psr_assoc {
> > @@ -43,6 +52,7 @@ struct psr_cmt *__read_mostly psr_cmt;
> >  
> >  static unsigned long *__read_mostly cat_socket_enable;
> >  static struct psr_cat_socket_info *__read_mostly cat_socket_info;
> > +static unsigned long *__read_mostly cdp_socket_enable;
> 
> ... this one then perhaps is redundency. If only one is need then I really
> like to keep the latter one.
> 

You're right. I would refine this.

> > @@ -470,6 +500,7 @@ static void cat_cpu_init(void)
> >  struct psr_cat_socket_info *info;
> >  unsigned int socket;
> >  unsigned int cpu = smp_processor_id();
> > +uint64_t val;
> >  const struct cpuinfo_x86 *c = cpu_data + cpu;
> >  
> >  if ( !cpu_has(c, X86_FEATURE_CAT) || c->cpuid_level < 
> > PSR_CPUID_LEVEL_CAT )
> > @@ -490,13 +521,34 @@ static void cat_cpu_init(void)
> >  info->cos_to_cbm = temp_cos_to_cbm;
> >  temp_cos_to_cbm = NULL;
> >  /* cos=0 is reserved as default cbm(all ones). */
> > -info->cos_to_cbm[0].cbm = (1ull << info->cbm_len) - 1;
> > +info->cos_to_cbm[0].u.cbm = (1ull << info->cbm_len) - 1;
> >  
> >  spin_lock_init(>cbm_lock);
> >  
> >  set_bit(socket, cat_socket_enable);
> >  printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, 
> > cbm_len:%u\n",
> > socket, info->cos_max, info->cbm_len);
> 
> If CDP is enable below, then this information is quite misleading.
> 

Is it proper to remove CAT printk info?

> > +
> > +if ( (ecx & PSR_CAT_CDP_CAPABILITY) && (opt_psr & PSR_CDP) )
> > +{
> > +if ( test_bit(socket, cdp_socket_enable) )
> > +return;
> > +
> > +rdmsrl(MSR_IA32_PSR_L3_QOS_CFG, val);
> > +wrmsrl(MSR_IA32_PSR_L3_QOS_CFG, val | 1 << 
> > PSR_L3_QOS_CDP_ENABLE_BIT);
> > +
> > +/* No need to write register since CBMs are fully open as 
> > default by HW */
> > +info->cos_to_cbm[0].u.cdp.code = (1ull << info->cbm_len) - 1;
> > +info->cos_to_cbm[0].u.cdp.data = (1ull << info->cbm_len) - 1;
> 
> If I remember correctly, HW is supposed to boot as CAT mode by default and
> only IA32_L3_QOS_Mask_0 is all ones, other mask msrs however are not mentioned
> in the spec which then may contain arbitrary value. So I guesss as
> least you need write all ones to IA32_L3_QOS_Mask_1 explicitly and that
> should be done before you enabled CDP.
> 

I have tested it at Broadwell server

Re: [Xen-devel] [PATCH v2 3/4] tools: add tools support for Intel CDP

[He Chen]

On Wed, Sep 09, 2015 at 03:32:11PM +0800, Chao Peng wrote:
> On Wed, Sep 09, 2015 at 01:16:47PM +0800, He Chen wrote:
> > This is the xl/xc changes to support Intel Code/Data Prioritization.
> > CAT xl commands to set/get CBMs are extended to support CDP.
> > 
> > Signed-off-by: He Chen <he.c...@linux.intel.com>
> > ---
> > diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
> > index 5f9047c..e4eb4df 100644
> > --- a/tools/libxl/libxl.h
> > +++ b/tools/libxl/libxl.h
> > @@ -796,6 +796,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, 
> > libxl_mac *src);
> >   * If this is defined, the Cache Allocation Technology feature is 
> > supported.
> >   */
> >  #define LIBXL_HAVE_PSR_CAT 1
> > +
> > +/*
> > + * LIBXL_HAVE_PSR_CDP
> > + *
> > + * If this is defined, the Code/Data Prioritization feature is supported.
> > + */
> > +#define LIBXL_HAVE_PSR_CDP 1
> >  #endif
> >  
> >  /*
> > @@ -1729,6 +1736,9 @@ int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, 
> > libxl_psr_cat_info **info,
> >  void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, int nr);
> >  #endif
> >  
> > +#ifdef LIBXL_HAVE_PSR_CDP
> > +#endif
> > +
> 
> Why this? There are several of them in this patch.
> 

My carelessness. I would remove the redundant code lines.

> >  /* misc */
> >  
> >  /* Each of these sets or clears the flag according to whether the
> > diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
> > index 3378239..26939a5 100644
> > --- a/tools/libxl/libxl_psr.c
> > +++ b/tools/libxl/libxl_psr.c
> > @@ -297,6 +297,7 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t 
> > domid,
> >  GC_INIT(ctx);
> >  int rc;
> >  int socketid, nr_sockets;
> > +libxl_psr_cat_info *info;
> >  
> >  rc = libxl__count_physical_sockets(gc, _sockets);
> >  if (rc) {
> > @@ -304,14 +305,41 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t 
> > domid,
> >  goto out;
> >  }
> >  
> > +rc = libxl_psr_cat_get_l3_info(ctx, , _sockets);
> > +if (rc) {
> > +LOGE(ERROR, "Failed to get cat info");
> > +goto out;
> > +}
> > +
> > +if (!info->cdp_enabled &&
> > +   (type == LIBXL_PSR_CBM_TYPE_L3_CODE ||
> > +type == LIBXL_PSR_CBM_TYPE_L3_DATA))
> > +{
> > +LOGE(ERROR, "Unable to set Code/Data CBM with CDP disabled");
> > +rc = EINVAL;
> > +free(info);
> > +goto out;
> > +}
> > +
> >  libxl_for_each_set_bit(socketid, *target_map) {
> >  if (socketid >= nr_sockets)
> >  break;
> > -if (xc_psr_cat_set_domain_data(ctx->xch, domid, type, socketid, 
> > cbm)) {
> > +if (info->cdp_enabled && type == LIBXL_PSR_CBM_TYPE_L3_CBM)
> > +{
> > +if(xc_psr_cat_set_domain_data(ctx->xch, domid,
> > +   LIBXL_PSR_CBM_TYPE_L3_CODE, socketid, cbm) ||
> > +   xc_psr_cat_set_domain_data(ctx->xch, domid,
> > +   LIBXL_PSR_CBM_TYPE_L3_DATA, socketid, cbm))
> > +{
> > +libxl__psr_cat_log_err_msg(gc, errno);
> > +rc = ERROR_FAIL;
> > +}
> 
> Will you merge the two if's?
> 
> Chao

Surely.

> > +} else if (xc_psr_cat_set_domain_data(ctx->xch, domid, type, 
> > socketid, cbm)) {
> >  libxl__psr_cat_log_err_msg(gc, errno);
> >  rc = ERROR_FAIL;
> >  }
> >  }
> > +free(info);
> >  
> >  out:
> >  GC_FREE;
> 
> ___
> Xen-devel mailing list
> Xen-devel@lists.xen.org
> http://lists.xen.org/xen-devel

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v2 1/4] x86: Support enable CDP by boot parameter and add get CDP status

[He Chen]

Intel Code/Data Prioritization(CDP) feature is based on CAT. cdp_enabled
is added to CAT socket info to indicate CDP is on or off on the socket,
note that cos_max would be half when CDP is on. struct psr_cat_cbm is
extended to support CDP operation. Extend psr_get_cat_l3_info sysctl to
get CDP status.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/psr.c  | 84 ++---
 xen/arch/x86/sysctl.c   |  5 ++-
 xen/include/asm-x86/msr-index.h |  3 ++
 xen/include/asm-x86/psr.h   | 11 +-
 xen/include/public/sysctl.h |  2 +
 5 files changed, 89 insertions(+), 16 deletions(-)

diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..ba0f726 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -17,13 +17,21 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
+#define PSR_CDP(1<<2)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+} cdp;
+} u;
 unsigned int ref;
 };
 
@@ -32,6 +40,7 @@ struct psr_cat_socket_info {
 unsigned int cos_max;
 struct psr_cat_cbm *cos_to_cbm;
 spinlock_t cbm_lock;
+bool_t cdp_enabled;
 };
 
 struct psr_assoc {
@@ -43,6 +52,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_enable;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -94,6 +104,7 @@ static void __init parse_psr_param(char *s)
 
 parse_psr_bool(s, val_str, "cmt", PSR_CMT);
 parse_psr_bool(s, val_str, "cat", PSR_CAT);
+parse_psr_bool(s, val_str, "cdp", PSR_CDP);
 
 if ( val_str && !strcmp(s, "rmid_max") )
 opt_rmid_max = simple_strtoul(val_str, NULL, 0);
@@ -262,7 +273,7 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 }
 
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint32_t *flags)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +283,10 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
 
+*flags = 0;
+if ( info->cdp_enabled )
+*flags |= PSR_CAT_FLAG_L3_CDP;
+
 return 0;
 }
 
@@ -282,7 +297,7 @@ int psr_get_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t *cbm)
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
 
 return 0;
 }
@@ -313,19 +328,34 @@ static bool_t psr_check_cbm(unsigned int cbm_len, 
uint64_t cbm)
 struct cos_cbm_info
 {
 unsigned int cos;
-uint64_t cbm;
+uint64_t cbm_code;
+uint64_t cbm_data;
+bool_t cdp;
 };
 
 static void do_write_l3_cbm(void *data)
 {
 struct cos_cbm_info *info = data;
 
-wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
+if ( info->cdp )
+{
+wrmsrl(MSR_IA32_PSR_L3_MASK_CBM_CODE(info->cos), info->cbm_code);
+wrmsrl(MSR_IA32_PSR_L3_MASK_CBM_DATA(info->cos), info->cbm_data);
+}
+else
+wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm_code);
 }
 
-static int write_l3_cbm(unsigned int socket, unsigned int cos, uint64_t cbm)
+static int write_l3_cbm(unsigned int socket, unsigned int cos,
+uint64_t cbm_code, uint64_t cbm_data, bool_t cdp)
 {
-struct cos_cbm_info info = { .cos = cos, .cbm = cbm };
+struct cos_cbm_info info =
+{
+  .cos = cos,
+  .cbm_code = cbm_code,
+  .cbm_data = cbm_data,
+  .cdp = cdp,
+};
 
 if ( socket == cpu_to_socket(smp_processor_id()) )
 do_write_l3_cbm();
@@ -363,7 +393,7 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t cbm)
 /* If still not found, then keep unused one. */
 if ( !found && cos != 0 && map[cos].ref == 0 )
 found = map + cos;
-else if ( map[cos].cbm == cbm )
+else if ( map[cos].u.cbm == cbm )
 {
 if ( unlikely(cos == old_cos) )
 {
@@ -387,16 +417,16 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t cbm)
 }
 
 cos = found - map;
-if ( found->cbm != cbm )
+if ( found->u.cbm != cbm )
 {
-int ret = write_l3_cbm(socket, cos, cbm);
+int ret = write_l3_cbm(socket, cos, cbm, 0, 0);
 
 if ( ret )
 {
 spin_unlock(>cbm_lock);
  

[Xen-devel] [PATCH v2 4/4] docs: add document to introduce CDP command

[He Chen]

Add CDP command in xl interface man page and add description of CDP
in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/man/xl.pod.1 | 14 ++
 docs/misc/xl-psr.markdown | 44 +++-
 2 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f22c3f3..3801039 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1530,6 +1530,12 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocation, which support specify code or data cache for
+applications. CDP is used on VM basis in the Xen implementation. To specify
+code or data CBM for the domain, CDP feature must be enabled and CBM type
+options need to be specified when setting CBM.
+
 =over 4
 
 =item B [I] I I
@@ -1545,6 +1551,14 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM with CDP enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM with CDP enabled.
+
 =back
 
 =item B [I]
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index 3545912..4c77737 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -14,7 +14,7 @@ tracks cache utilization of memory accesses according to the 
RMID and reports
 monitored data via a counter register.
 
 For more detailed information please refer to Intel SDM chapter
-"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+"17.15 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
 
 In Xen's implementation, each domain in the system can be assigned a RMID
 independently, while RMID=0 is reserved for monitoring domains that don't
@@ -91,17 +91,44 @@ For example, assuming a system with 8 portions and 3 
domains:
first domain exclusive access to half the cache, and the other two exclusive
access to one quarter each.
 
-For more detailed information please refer to Intel SDM chapter
-"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
-
 In Xen's implementation, CBM can be configured with libxl/xl interfaces but
 COS is maintained in hypervisor only. The cache partition granularity is per
 domain, each domain has COS=0 assigned by default, the corresponding CBM is
 all-ones, which means all the cache resource can be used by default.
 
+Code/Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cache capacity to the characteristics of the workload. CDP extends
+Cache Allocation Technology (CAT) by providing separate code and data masks
+per Class of Service (COS).
+
+CDP is disabled on the processor by default. If the CAT MSRs are used without
+enabling CDP, the processor operates in a traditional CAT-only mode.
+
+When CDP is enabled,
+
+ * the CAT mask MSRs are re-mapped into interleaved pairs of mask MSRs for
+   data or code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows OS or Hypervisor to partition cache allocation more fine-grained,
+code cache and data cache can be specified respectively. To enable CDP on
+platform, all sockets in the platform must have CDP either enabled or
+disabled, not a mix. With CDP enabled, one COS corresponds to two CBMs (code
+CBM & data CBM), which means the number of available COS will reduce to half
+when CDP is on.
+
+For more detailed information please refer to Intel SDM chapter
+"17.16 - Platform Shared Resource Control: Cache Allocation Technology".
+
 ### xl interfaces
 
-System CAT information such as maximum COS and CBM length can be obtained by:
+System CAT information such as maximum COS, CBM length and CDP status can be
+obtained by:
 
 `xl psr-hwinfo --cat`
 
@@ -119,6 +146,13 @@ A cbm is valid only when:
 In a multi-socket system, the same cbm will be set on each socket by default.
 Per socket cbm can be specified with the `--socket SOCKET` option.
 
+To specify code CBM for the domain, `-c` or `--code` option is needed.
+
+To specify data CBM for the domain, `-d` or `--data` option is needed.
+
+If neither `-c` nor `-d` option is specified when CDP is on, the same code CBM
+and data CBM will be set for the domain.
+
 Setting the CBM may not be successful if insufficient COS is available. In
 such case unused COS(es) may be freed by setting CBM of all related domains to
 its default value(all-ones).
-- 
1.9.1


_

[Xen-devel] [PATCH v2 0/4] Intel Code/Data Prioritization(CDP) feature enabling

[He Chen]

Changes in v2:
- x86: Enable CDP by boot parameter instead of enabling/disabling CDP at
runtime (suggested by Andrew)
- tools: remove psr-cat-cdp-enable/disable xl commands
- code style

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, and extends
CBM operation functions to support CDP. For all the changes, please see in
each patch.

This patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks

He Chen (4):
  x86: Support enable CDP by boot parameter and add get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools: add tools support for Intel CDP
  docs: add document to introduce CDP command

 docs/man/xl.pod.1   |  14 +++
 docs/misc/xl-psr.markdown   |  44 +++-
 tools/libxc/include/xenctrl.h   |   7 +-
 tools/libxc/xc_psr.c|  17 ++-
 tools/libxl/libxl.h |  10 ++
 tools/libxl/libxl_psr.c |  32 +-
 tools/libxl/libxl_types.idl |   3 +
 tools/libxl/xl.h|   2 +
 tools/libxl/xl_cmdimpl.c|  52 +++--
 tools/libxl/xl_cmdtable.c   |   5 +
 xen/arch/x86/domctl.c   |  32 +-
 xen/arch/x86/psr.c  | 237 
 xen/arch/x86/sysctl.c   |   5 +-
 xen/include/asm-x86/msr-index.h |   3 +
 xen/include/asm-x86/psr.h   |  23 +++-
 xen/include/public/domctl.h |   4 +
 xen/include/public/sysctl.h |   2 +
 17 files changed, 420 insertions(+), 72 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH v2 3/4] tools: add tools support for Intel CDP

[He Chen]

This is the xl/xc changes to support Intel Code/Data Prioritization.
CAT xl commands to set/get CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 tools/libxc/include/xenctrl.h |  7 --
 tools/libxc/xc_psr.c  | 17 +-
 tools/libxl/libxl.h   | 10 +
 tools/libxl/libxl_psr.c   | 32 --
 tools/libxl/libxl_types.idl   |  3 +++
 tools/libxl/xl.h  |  2 ++
 tools/libxl/xl_cmdimpl.c  | 52 ++-
 tools/libxl/xl_cmdtable.c |  5 +
 8 files changed, 113 insertions(+), 15 deletions(-)

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index de3c0ad..54f2069 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2798,7 +2798,9 @@ enum xc_psr_cmt_type {
 typedef enum xc_psr_cmt_type xc_psr_cmt_type;
 
 enum xc_psr_cat_type {
-XC_PSR_CAT_L3_CBM = 1,
+XC_PSR_CAT_L3_CBM  = 1,
+XC_PSR_CAT_L3_CODE = 2,
+XC_PSR_CAT_L3_DATA = 3,
 };
 typedef enum xc_psr_cat_type xc_psr_cat_type;
 
@@ -2824,7 +2826,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, 
uint32_t domid,
xc_psr_cat_type type, uint32_t target,
uint64_t *data);
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len);
+   uint32_t *cos_max, uint32_t *cbm_len,
+   bool *cdp_enabled);
 #endif
 
 #endif /* XENCTRL_H */
diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index d8b3a51..5bbe950 100644
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -260,6 +260,12 @@ int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -287,6 +293,12 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -306,7 +318,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 }
 
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len)
+   uint32_t *cos_max, uint32_t *cbm_len,
+   bool *cdp_enabled)
 {
 int rc;
 DECLARE_SYSCTL;
@@ -320,6 +333,8 @@ int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t 
socket,
 {
 *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
 *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+*cdp_enabled = sysctl.u.psr_cat_op.u.l3_info.flags &
+   XEN_SYSCTL_PSR_CAT_L3_CDP;
 }
 
 return rc;
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5f9047c..e4eb4df 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -796,6 +796,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, 
libxl_mac *src);
  * If this is defined, the Cache Allocation Technology feature is supported.
  */
 #define LIBXL_HAVE_PSR_CAT 1
+
+/*
+ * LIBXL_HAVE_PSR_CDP
+ *
+ * If this is defined, the Code/Data Prioritization feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CDP 1
 #endif
 
 /*
@@ -1729,6 +1736,9 @@ int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, 
libxl_psr_cat_info **info,
 void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, int nr);
 #endif
 
+#ifdef LIBXL_HAVE_PSR_CDP
+#endif
+
 /* misc */
 
 /* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 3378239..26939a5 100644
--- a/tools/libxl/libxl_psr.c
+++ b/tools/libxl/libxl_psr.c
@@ -297,6 +297,7 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
 GC_INIT(ctx);
 int rc;
 int socketid, nr_sockets;
+libxl_psr_cat_info *info;
 
 rc = libxl__count_physical_sockets(gc, _sockets);
 if (rc) {
@@ -304,14 +305,41 @@ int libxl_psr_cat_set_cbm(libxl_ctx *ctx, uint32_t domid,
 goto out;
 }
 
+rc = libxl_psr_cat_get_l3_info(ctx, , _sockets);
+if (rc) {
+LOGE(ERROR, "Failed to get cat info");
+goto out;
+}
+
+if (!info->cdp_enabled &&
+   (type == LIBXL_PSR_CBM_TYPE_L3_CODE ||
+type == LIBXL_PSR_CBM_TYPE_L3_DATA))
+{
+LOGE(ERROR, "Unable to set Code/Data CBM with CDP 

[Xen-devel] [PATCH v2 2/4] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

CDP extends CAT and provides the capacity to control L3 code & data
cache. With CDP, one COS correspond to two CMBs(code & data). cbm_type
is added to support distinguish different CBM operation. Besides, new
domctl cmds are introdunced to support set/get CDP CBM. Some CAT
functions to operation CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/domctl.c   |  32 -
 xen/arch/x86/psr.c  | 163 ++--
 xen/include/asm-x86/psr.h   |  12 +++-
 xen/include/public/domctl.h |   4 ++
 4 files changed, 170 insertions(+), 41 deletions(-)

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index bf62a88..734fddb 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1167,12 +1167,40 @@ long arch_do_domctl(
 {
 case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
 ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
- domctl->u.psr_cat_op.data);
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 break;
 
 case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
 ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
- >u.psr_cat_op.data);
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 copyback = 1;
 break;
 
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index ba0f726..c7d8356 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -290,14 +290,32 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 return 0;
 }
 
-int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+int psr_get_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t *cbm, enum cbm_type type)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+if ( type == PSR_CBM_TYPE_L3 )
+{
+if ( info->cdp_enabled )
+return -EXDEV;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+}
+else if ( type == PSR_CBM_TYPE_L3_CODE )
+{
+if ( !info->cdp_enabled )
+return -EXDEV;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.code;
+}
+else
+{
+if ( !info->cdp_enabled )
+return -EXDEV;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.data;
+}
 
 return 0;
 }
@@ -371,65 +389,136 @@ static int write_l3_cbm(unsigned int socket, unsigned 
int cos,
 return 0;
 }
 
-int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
+static int find_cos(struct psr_cat_cbm *map, int cos_max,
+uint64_t cbm_code, uint64_t cbm_data, bool_t cdp_enabled)
 {
-unsigned int old_cos, cos;
-struct psr_cat_cbm *map, *found = NULL;
+unsigned int cos;
+
+if ( !cdp_enabled )
+{
+for ( cos = 0; cos <= cos_max; cos++ )
+if ( map[cos].ref && map[cos].u.cbm == cbm_code )
+return cos;
+}
+else
+{
+for ( cos = 0; cos <= cos_max; cos++ )
+if ( map[cos].ref && map[cos].u.cdp.code == cbm_code &&
+ map[cos].u.cdp.data == cbm_data )
+return cos;
+}
+
+return -ENOENT;
+}
+
+static int pick_avail_cos(struct psr_cat_cbm *map, int cos_max, int old_cos)
+{
+int cos;
+
+/* If old cos is referred only by the domain, then use it. */
+if ( map[old_cos].ref == 1 )
+return old_cos;
+
+/* Then we pick an unused one, never pick 0 */
+for ( cos =

Re: [Xen-devel] [PATCH 3/5] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

On Wed, Sep 02, 2015 at 12:59:07PM +0100, Andrew Cooper wrote:
> On 02/09/15 09:28, He Chen wrote:
> > CDP extends CAT and provides the capacity to control L3 code & data
> > cache. With CDP, one COS correspond to two CMBs(code & data). cbm_type
> > is added to support distinguish different CBM operation. Besides, new
> > domctl cmds are introdunced to support set/get CDP CBM. Some CAT
> > functions to operation CBMs are extended to support CDP.
> >
> > Signed-off-by: He Chen <he.c...@linux.intel.com>
> > ---
> >  xen/arch/x86/domctl.c   |  33 +-
> >  xen/arch/x86/psr.c  | 142 
> > 
> >  xen/include/asm-x86/psr.h   |  12 +++-
> >  xen/include/public/domctl.h |   4 ++
> >  4 files changed, 150 insertions(+), 41 deletions(-)
> >
> > diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
> > index 26596dd..8e92d24 100644
> > --- a/xen/arch/x86/psr.c
> > +++ b/xen/arch/x86/psr.c
> > +static int pick_avail_cos(struct psr_cat_cbm *map, int cos_max, int 
> > old_cos)
> > +{
> > +int cos;
> > +
> > +/* If old cos is referred only by the domain, then use it. */
> > +if ( map[old_cos].ref == 1 )
> > +return old_cos;
> > +
> > +/* Then we pick an unused one, never pick 0 */
> > +for ( cos = 1; cos <= cos_max; cos++ )
> > +if ( map[cos].ref == 0 )
> > +return cos;
> > +
> > +return -EOVERFLOW;
> 
> ENOENT surely, or use EOVERFLOW consistently.
> 

I am not sure I got your point here. pick_avail_cos is to get an unused
COS, if succeed, it returns a positive number which means COS, but
when fail, it should return a negative number to indicate an error.

As far as I know, ENOENT is 2 and EOVERFLOW is 75, if I return ENOENT
directly, the function which call pick_avail_cos could not tell the
value is a valid COS or an error number.

Would you mind explaining in more detail for me and thanks.

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

Re: [Xen-devel] [PATCH 0/5] Intel Code/Data Prioritization(CDP) feature enabling

[He Chen]

On Wed, Sep 02, 2015 at 01:08:33PM +0100, Andrew Cooper wrote:
> On 02/09/15 09:27, He Chen wrote:
> > Hi all,
> >
> > Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
> > platforms, which is an extension of CAT. CDP enables isolation and separate
> > prioritization of code and data fetches to the L3 cache in a software
> > configurable manner, which can enable workload prioritization and tuning of
> > cache capacity to the characteristics of the workload. CDP extends Cache
> > Allocation Technology (CAT) by providing separate code and data capacity bit
> > masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
> > implementation.
> >
> > More information about CDP, please refer to Intel SDM, Volumn 3, section 
> > 17.16
> > http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf
> >
> > This patch series enables CDP feature in Xen based on CAT code, including
> > extending CBM operation functions and introducing new commands to 
> > enable/disable
> > CDP dynamically. For all the changes, please see in each patch.
> >
> > This patchset has been tested on Intel Broadwell server platform.
> >
> > To make this patchset better, any comment or suggestion is welcomed, I would
> > really appreciate it.
> 
> I have taken a look at patches 1-3.  For the most part, it looks good.
> 
> The main point I have is on patch 2, as to whether it is sensible to
> permit enabling/disabling cdp at runtime.  I suggest that it is not
> sensible, and should be a command line parameter instead.
> 
> If this is agreed as ok going forwards, patches 3 through 5 should
> become rather more simple.
> 
> ~Andrew

Thanks for your patient review and valuable suggestions.

About permitting enabling/disabling CDP at runtime, I agree with you to
use command line parameter instead, it really makes code simple and
reliable.

For caution's sake, hardware support confingure CDP dynamically at any
point during normal system operation according to Intel SDM (see section
17.16.2), and that is why I wrote patch 2.

Anyway, since there is few cases to change CDP at runtime, I think it is
better to make this a boot-time parameter. I would resend v2 patch soon
and thanks again~

___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH 2/5] x86: Support enable/disable CDP dynamically and get CDP status

[He Chen]

 cdp_enabled is added to CAT socket info to indicate CDP is on or off on
 the socket and struct psr_cat_cbm is extended to support CDP operation.
 IA32_L3_QOS_CFG is a new MSR to enable/disable L3 CDP, when enable or
 disable CDP, all domains will reset to COS0 with fully access all L3
 cache.

 Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/psr.c  | 164 
 xen/arch/x86/sysctl.c   |   9 ++-
 xen/include/asm-x86/psr.h   |  10 ++-
 xen/include/public/sysctl.h |   5 ++
 4 files changed, 174 insertions(+), 14 deletions(-)

diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index b357816..26596dd 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -17,13 +17,20 @@
 #include 
 #include 
 #include 
+#include 
 #include 
 
 #define PSR_CMT(1<<0)
 #define PSR_CAT(1<<1)
 
 struct psr_cat_cbm {
-uint64_t cbm;
+union {
+uint64_t cbm;
+struct {
+uint64_t code;
+uint64_t data;
+}cdp;
+}u;
 unsigned int ref;
 };
 
@@ -32,6 +39,7 @@ struct psr_cat_socket_info {
 unsigned int cos_max;
 struct psr_cat_cbm *cos_to_cbm;
 spinlock_t cbm_lock;
+bool_t cdp_enabled;
 };
 
 struct psr_assoc {
@@ -263,7 +271,7 @@ static struct psr_cat_socket_info 
*get_cat_socket_info(unsigned int socket)
 }
 
 int psr_get_cat_l3_info(unsigned int socket, uint32_t *cbm_len,
-uint32_t *cos_max)
+uint32_t *cos_max, uint8_t *cdp_enabled)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
@@ -272,6 +280,7 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 
 *cbm_len = info->cbm_len;
 *cos_max = info->cos_max;
+*cdp_enabled = (uint8_t)info->cdp_enabled;
 
 return 0;
 }
@@ -283,7 +292,7 @@ int psr_get_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t *cbm)
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].cbm;
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
 
 return 0;
 }
@@ -314,19 +323,33 @@ static bool_t psr_check_cbm(unsigned int cbm_len, 
uint64_t cbm)
 struct cos_cbm_info
 {
 unsigned int cos;
-uint64_t cbm;
+uint64_t cbm_code;
+uint64_t cbm_data;
+bool_t cdp_mode;
 };
 
 static void do_write_l3_cbm(void *data)
 {
 struct cos_cbm_info *info = data;
 
-wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm);
+if ( info->cdp_mode == 0 )
+wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos), info->cbm_code);
+else
+{
+wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos*2+1), info->cbm_code);
+wrmsrl(MSR_IA32_PSR_L3_MASK(info->cos*2), info->cbm_data);
+}
 }
 
-static int write_l3_cbm(unsigned int socket, unsigned int cos, uint64_t cbm)
+static int write_l3_cbm(unsigned int socket, unsigned int cos,
+uint64_t cbm_code, uint64_t cbm_data, bool_t cdp_mode)
 {
-struct cos_cbm_info info = { .cos = cos, .cbm = cbm };
+struct cos_cbm_info info =
+{ .cos = cos,
+  .cbm_code = cbm_code,
+  .cbm_data = cbm_data,
+  .cdp_mode = cdp_mode,
+};
 
 if ( socket == cpu_to_socket(smp_processor_id()) )
 do_write_l3_cbm();
@@ -364,7 +387,7 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t cbm)
 /* If still not found, then keep unused one. */
 if ( !found && cos != 0 && map[cos].ref == 0 )
 found = map + cos;
-else if ( map[cos].cbm == cbm )
+else if ( map[cos].u.cbm == cbm )
 {
 if ( unlikely(cos == old_cos) )
 {
@@ -388,16 +411,16 @@ int psr_set_l3_cbm(struct domain *d, unsigned int socket, 
uint64_t cbm)
 }
 
 cos = found - map;
-if ( found->cbm != cbm )
+if ( found->u.cbm != cbm )
 {
-int ret = write_l3_cbm(socket, cos, cbm);
+int ret = write_l3_cbm(socket, cos, cbm, 0, 0);
 
 if ( ret )
 {
 spin_unlock(>cbm_lock);
 return ret;
 }
-found->cbm = cbm;
+found->u.cbm = cbm;
 }
 
 found->ref++;
@@ -491,7 +514,7 @@ static void cat_cpu_init(void)
 info->cos_to_cbm = temp_cos_to_cbm;
 temp_cos_to_cbm = NULL;
 /* cos=0 is reserved as default cbm(all ones). */
-info->cos_to_cbm[0].cbm = (1ull << info->cbm_len) - 1;
+info->cos_to_cbm[0].u.cbm = (1ull << info->cbm_len) - 1;
 
 spin_lock_init(>cbm_lock);
 
@@ -556,6 +579,123 @@ static int psr_cpu_prepare(unsigned int cpu)
 return cat_cpu_prepare(cpu);
 }
 
+static void do_write_l3_config(void *data)
+{
+uint64_t val;
+uint64_t *mask = data;
+
+rdmsrl(PSR_L3_QOS_CFG, val);
+wrmsrl(PSR_L3_QOS_CFG, val | *mask);
+}
+
+static int config_socket_l3_cdp(unsigned int socket

[Xen-devel] [PATCH 5/5] docs: add document to introduce CDP command

[He Chen]

Add CDP command in xl interface man page and add description of CDP
in xl-psr.markdown.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 docs/man/xl.pod.1 | 22 +++
 docs/misc/xl-psr.markdown | 56 ++-
 2 files changed, 73 insertions(+), 5 deletions(-)

diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1
index f22c3f3..3d7bde6 100644
--- a/docs/man/xl.pod.1
+++ b/docs/man/xl.pod.1
@@ -1530,6 +1530,12 @@ applications. In the Xen implementation, CAT is used to 
control cache allocation
 on VM basis. To enforce cache on a specific domain, just set capacity bitmasks
 (CBM) for the domain.
 
+Intel Broadwell and later server platforms also offer Code/Data Prioritization
+(CDP) for cache allocation, which support specify code or data cache for
+applications. CDP is used on VM basis in the Xen implementation. To specify
+code or data CBM for the domain, CDP feature must be enabled and CBM type
+options need to be specified when setting CBM.
+
 =over 4
 
 =item B [I] I I
@@ -1545,12 +1551,28 @@ B
 
 Specify the socket to process, otherwise all sockets are processed.
 
+=item B<-c>, B<--code>
+
+Set code CBM with CDP enabled.
+
+=item B<-d>, B<--data>
+
+Set data CBM with CDP enabled.
+
 =back
 
 =item B [I]
 
 Show CAT settings for a certain domain or all domains.
 
+=item B
+
+Enable Code/Data Prioritization.
+
+=item B
+
+Disable Code/Data Prioritization.
+
 =back
 
 =head1 TO BE DOCUMENTED
diff --git a/docs/misc/xl-psr.markdown b/docs/misc/xl-psr.markdown
index 3545912..a3728a9 100644
--- a/docs/misc/xl-psr.markdown
+++ b/docs/misc/xl-psr.markdown
@@ -14,7 +14,7 @@ tracks cache utilization of memory accesses according to the 
RMID and reports
 monitored data via a counter register.
 
 For more detailed information please refer to Intel SDM chapter
-"17.14 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
+"17.15 - Platform Shared Resource Monitoring: Cache Monitoring Technology".
 
 In Xen's implementation, each domain in the system can be assigned a RMID
 independently, while RMID=0 is reserved for monitoring domains that don't
@@ -91,17 +91,48 @@ For example, assuming a system with 8 portions and 3 
domains:
first domain exclusive access to half the cache, and the other two exclusive
access to one quarter each.
 
-For more detailed information please refer to Intel SDM chapter
-"17.15 - Platform Shared Resource Control: Cache Allocation Technology".
-
 In Xen's implementation, CBM can be configured with libxl/xl interfaces but
 COS is maintained in hypervisor only. The cache partition granularity is per
 domain, each domain has COS=0 assigned by default, the corresponding CBM is
 all-ones, which means all the cache resource can be used by default.
 
+Code/Data Prioritization (CDP) Technology is an extension of CAT, which is
+available on Intel Broadwell and later server platforms. CDP enables isolation
+and separate prioritization of code and data fetches to the L3 cache in a
+software configurable manner, which can enable workload prioritization and
+tuning of cache capacity to the characteristics of the workload. CDP extends
+Cache Allocation Technology (CAT) by providing separate code and data masks
+per Class of Service (COS).
+
+CDP is disabled on the processor by default. If the CAT MSRs are used without
+enabling CDP, the processor operates in a traditional CAT-only mode.
+
+When CDP is enabled,
+
+ * the CAT mask MSRs are re-mapped into interleaved pairs of mask MSRs for
+   data or code fetches.
+
+ * the range of COS for CAT is re-indexed, with the lower-half of the COS
+   range available for CDP.
+
+CDP allows OS or
+Hypervisor to partition cache allocation more fine-grained, code cache and
+data cache can be specified respectively. To enable CDP on platform, all
+sockets in the platform must have CDP either enabled or disabled, not a mix.
+With CDP enabled, one COS corresponds to two CBMs(code CBM & data CBM),
+which means the number of available COS will reduce to half when CDP on.
+
+Further more, if enabling/disabling CDP dynamically on runtime, all domains
+are reset to COS[0] with fully access to L3 cache before enabling or disabling
+CDP.
+
+For more detailed information please refer to Intel SDM chapter
+"17.16 - Platform Shared Resource Control: Cache Allocation Technology".
+
 ### xl interfaces
 
-System CAT information such as maximum COS and CBM length can be obtained by:
+System CAT information such as maximum COS, CBM length and CDP status can be
+obtained by:
 
 `xl psr-hwinfo --cat`
 
@@ -119,6 +150,13 @@ A cbm is valid only when:
 In a multi-socket system, the same cbm will be set on each socket by default.
 Per socket cbm can be specified with the `--socket SOCKET` option.
 
+To specify code CBM for the domain, `-c` or `--code` option is needed.
+
+To specify data CBM for the domain, `-d` or `--data` 

[Xen-devel] [PATCH 0/5] Intel Code/Data Prioritization(CDP) feature enabling

[He Chen]

Hi all,

Code/Data Prioritization(CDP) is offered in Intel Broadwell and later server
platforms, which is an extension of CAT. CDP enables isolation and separate
prioritization of code and data fetches to the L3 cache in a software
configurable manner, which can enable workload prioritization and tuning of
cache capacity to the characteristics of the workload. CDP extends Cache
Allocation Technology (CAT) by providing separate code and data capacity bit
masks(CBM) per Class of Service (COS). CDP is used on VM basis in the Xen
implementation.

More information about CDP, please refer to Intel SDM, Volumn 3, section 17.16
http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-manual-325462.pdf

This patch series enables CDP feature in Xen based on CAT code, including
extending CBM operation functions and introducing new commands to enable/disable
CDP dynamically. For all the changes, please see in each patch.

This patchset has been tested on Intel Broadwell server platform.

To make this patchset better, any comment or suggestion is welcomed, I would
really appreciate it.

Thanks

He Chen (5):
  x86: detect Intel CDP feature
  x86: Support enable/disable CDP dynamically and get CDP status
  x86: add domctl cmd to set/get CDP code/data CBM
  tools: add tools support for Intel CDP
  docs: add document to introduce CDP command

 docs/man/xl.pod.1 |  22 +++
 docs/misc/xl-psr.markdown |  56 +++-
 tools/libxc/include/xenctrl.h |  10 +-
 tools/libxc/xc_psr.c  |  42 +-
 tools/libxl/libxl.h   |  12 ++
 tools/libxl/libxl_psr.c   |  64 -
 tools/libxl/libxl_types.idl   |   3 +
 tools/libxl/xl.h  |   4 +
 tools/libxl/xl_cmdimpl.c  |  81 +--
 tools/libxl/xl_cmdtable.c |  15 ++
 xen/arch/x86/domctl.c |  33 -
 xen/arch/x86/psr.c| 309 --
 xen/arch/x86/sysctl.c |   9 +-
 xen/include/asm-x86/psr.h |  26 +++-
 xen/include/public/domctl.h   |   4 +
 xen/include/public/sysctl.h   |   5 +
 16 files changed, 625 insertions(+), 70 deletions(-)

-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH 1/5] x86: detect Intel CDP feature

[He Chen]

Detect Intel Code/Data Prioritization(CDP) feature and store cpuid
information for later use. CDP feature is based on CAT, note that all
sockets in the platform must have CDP either enabled or disabled, not
a mix. cdp_socket_avail saves CDP capability of every socket so that
we can determine if CDP is supported in the platform.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/psr.c| 13 -
 xen/include/asm-x86/psr.h |  4 
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index c0daa2e..b357816 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -43,6 +43,7 @@ struct psr_cmt *__read_mostly psr_cmt;
 
 static unsigned long *__read_mostly cat_socket_enable;
 static struct psr_cat_socket_info *__read_mostly cat_socket_info;
+static unsigned long *__read_mostly cdp_socket_avail;
 
 static unsigned int __initdata opt_psr;
 static unsigned int __initdata opt_rmid_max = 255;
@@ -498,6 +499,13 @@ static void cat_cpu_init(void)
 printk(XENLOG_INFO "CAT: enabled on socket %u, cos_max:%u, 
cbm_len:%u\n",
socket, info->cos_max, info->cbm_len);
 }
+
+if ( ecx & PSR_CAT_CDP_CAPABILITY )
+{
+set_bit(socket, cdp_socket_avail);
+printk(XENLOG_INFO "CDP: available on socket %u\n",  socket);
+}
+
 }
 
 static void cat_cpu_fini(unsigned int cpu)
@@ -523,6 +531,8 @@ static void __init psr_cat_free(void)
 cat_socket_enable = NULL;
 xfree(cat_socket_info);
 cat_socket_info = NULL;
+xfree(cdp_socket_avail);
+cdp_socket_avail = NULL;
 }
 
 static void __init init_psr_cat(void)
@@ -535,8 +545,9 @@ static void __init init_psr_cat(void)
 
 cat_socket_enable = xzalloc_array(unsigned long, 
BITS_TO_LONGS(nr_sockets));
 cat_socket_info = xzalloc_array(struct psr_cat_socket_info, nr_sockets);
+cdp_socket_avail = xzalloc_array(unsigned long, BITS_TO_LONGS(nr_sockets));
 
-if ( !cat_socket_enable || !cat_socket_info )
+if ( !cat_socket_enable || !cat_socket_info || !cdp_socket_avail )
 psr_cat_free();
 }
 
diff --git a/xen/include/asm-x86/psr.h b/xen/include/asm-x86/psr.h
index 081750f..a6b83df 100644
--- a/xen/include/asm-x86/psr.h
+++ b/xen/include/asm-x86/psr.h
@@ -27,6 +27,10 @@
 /* L3 Monitoring Features */
 #define PSR_CMT_L3_OCCUPANCY   0x1
 
+/* CDP Capability */
+#define PSR_CAT_CDP_CAPABILITY   0x4
+
+
 struct psr_cmt_l3 {
 unsigned int features;
 unsigned int upscaling_factor;
-- 
1.9.1


___
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Xen-devel] [PATCH 4/5] tools: add tools support for Intel CDP

[He Chen]

This is the xc/xl changes to support Intel Code/Data Prioritization.
Two new xl commands are introduced to enable/disable CDP dynamically,
and CAT xl commands to set/get CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 tools/libxc/include/xenctrl.h | 10 --
 tools/libxc/xc_psr.c  | 42 +-
 tools/libxl/libxl.h   | 12 +++
 tools/libxl/libxl_psr.c   | 64 +-
 tools/libxl/libxl_types.idl   |  3 ++
 tools/libxl/xl.h  |  4 +++
 tools/libxl/xl_cmdimpl.c  | 81 +--
 tools/libxl/xl_cmdtable.c | 15 
 8 files changed, 217 insertions(+), 14 deletions(-)

diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h
index de3c0ad..665e6bd 100644
--- a/tools/libxc/include/xenctrl.h
+++ b/tools/libxc/include/xenctrl.h
@@ -2798,7 +2798,10 @@ enum xc_psr_cmt_type {
 typedef enum xc_psr_cmt_type xc_psr_cmt_type;
 
 enum xc_psr_cat_type {
-XC_PSR_CAT_L3_CBM = 1,
+XC_PSR_CAT_L3_CBM  = 1,
+XC_PSR_CAT_L3_CODE = 2,
+XC_PSR_CAT_L3_DATA = 3,
+
 };
 typedef enum xc_psr_cat_type xc_psr_cat_type;
 
@@ -2824,7 +2827,10 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, 
uint32_t domid,
xc_psr_cat_type type, uint32_t target,
uint64_t *data);
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len);
+   uint32_t *cos_max, uint32_t *cbm_len,
+   uint8_t *cdp_enabled);
+int xc_psr_cat_enable_cdp(xc_interface *xch);
+int xc_psr_cat_disable_cdp(xc_interface *xch);
 #endif
 
 #endif /* XENCTRL_H */
diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c
index d8b3a51..d4ff6f6 100644
--- a/tools/libxc/xc_psr.c
+++ b/tools/libxc/xc_psr.c
@@ -260,6 +260,12 @@ int xc_psr_cat_set_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -287,6 +293,12 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 case XC_PSR_CAT_L3_CBM:
 cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM;
 break;
+case XC_PSR_CAT_L3_CODE:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE;
+break;
+case XC_PSR_CAT_L3_DATA:
+cmd = XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA;
+break;
 default:
 errno = EINVAL;
 return -1;
@@ -306,7 +318,8 @@ int xc_psr_cat_get_domain_data(xc_interface *xch, uint32_t 
domid,
 }
 
 int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t socket,
-   uint32_t *cos_max, uint32_t *cbm_len)
+   uint32_t *cos_max, uint32_t *cbm_len,
+   uint8_t *cdp_enabled)
 {
 int rc;
 DECLARE_SYSCTL;
@@ -320,11 +333,38 @@ int xc_psr_cat_get_l3_info(xc_interface *xch, uint32_t 
socket,
 {
 *cos_max = sysctl.u.psr_cat_op.u.l3_info.cos_max;
 *cbm_len = sysctl.u.psr_cat_op.u.l3_info.cbm_len;
+*cdp_enabled = sysctl.u.psr_cat_op.u.l3_info.cdp_enabled;
 }
 
 return rc;
 }
 
+int xc_psr_cat_enable_cdp(xc_interface *xch)
+{
+int rc;
+DECLARE_SYSCTL;
+
+sysctl.cmd = XEN_SYSCTL_psr_cat_op;
+sysctl.u.psr_cat_op.cmd = XEN_SYSCTL_PSR_CAT_enable_cdp;
+
+rc = do_sysctl(xch, );
+
+return rc;
+}
+
+int xc_psr_cat_disable_cdp(xc_interface *xch)
+{
+int rc;
+DECLARE_SYSCTL;
+
+sysctl.cmd = XEN_SYSCTL_psr_cat_op;
+sysctl.u.psr_cat_op.cmd = XEN_SYSCTL_PSR_CAT_disable_cdp;
+
+rc = do_sysctl(xch, );
+
+return rc;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 5f9047c..68c14fb 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -796,6 +796,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, 
libxl_mac *src);
  * If this is defined, the Cache Allocation Technology feature is supported.
  */
 #define LIBXL_HAVE_PSR_CAT 1
+
+/*
+ * LIBXL_HAVE_PSR_CDP
+ *
+ * If this is defined, the Code/Data Prioritization feature is supported.
+ */
+#define LIBXL_HAVE_PSR_CDP 1
 #endif
 
 /*
@@ -1729,6 +1736,11 @@ int libxl_psr_cat_get_l3_info(libxl_ctx *ctx, 
libxl_psr_cat_info **info,
 void libxl_psr_cat_info_list_free(libxl_psr_cat_info *list, int nr);
 #endif
 
+#ifdef LIBXL_HAVE_PSR_CDP
+int libxl_psr_cat_enable_cdp(libxl_ctx *ctx);
+int libxl_psr_cat_disable_cdp(libxl_ctx *ctx);
+#endif
+
 /* misc */
 
 /* Each of these sets or clears the flag according to whether the
diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c
index 3378239..4dae5e5 100644
--- a/tools

[Xen-devel] [PATCH 3/5] x86: add domctl cmd to set/get CDP code/data CBM

[He Chen]

CDP extends CAT and provides the capacity to control L3 code & data
cache. With CDP, one COS correspond to two CMBs(code & data). cbm_type
is added to support distinguish different CBM operation. Besides, new
domctl cmds are introdunced to support set/get CDP CBM. Some CAT
functions to operation CBMs are extended to support CDP.

Signed-off-by: He Chen <he.c...@linux.intel.com>
---
 xen/arch/x86/domctl.c   |  33 +-
 xen/arch/x86/psr.c  | 142 
 xen/include/asm-x86/psr.h   |  12 +++-
 xen/include/public/domctl.h |   4 ++
 4 files changed, 150 insertions(+), 41 deletions(-)

diff --git a/xen/arch/x86/domctl.c b/xen/arch/x86/domctl.c
index bf62a88..c2d771e 100644
--- a/xen/arch/x86/domctl.c
+++ b/xen/arch/x86/domctl.c
@@ -1167,12 +1167,41 @@ long arch_do_domctl(
 {
 case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CBM:
 ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
- domctl->u.psr_cat_op.data);
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
 break;
 
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_CODE:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_SET_L3_DATA:
+ret = psr_set_l3_cbm(d, domctl->u.psr_cat_op.target,
+ domctl->u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
+break;
+
+
 case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CBM:
 ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
- >u.psr_cat_op.data);
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_CODE:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_CODE);
+copyback = 1;
+break;
+
+case XEN_DOMCTL_PSR_CAT_OP_GET_L3_DATA:
+ret = psr_get_l3_cbm(d, domctl->u.psr_cat_op.target,
+ >u.psr_cat_op.data,
+ PSR_CBM_TYPE_L3_DATA);
 copyback = 1;
 break;
 
diff --git a/xen/arch/x86/psr.c b/xen/arch/x86/psr.c
index 26596dd..8e92d24 100644
--- a/xen/arch/x86/psr.c
+++ b/xen/arch/x86/psr.c
@@ -285,14 +285,20 @@ int psr_get_cat_l3_info(unsigned int socket, uint32_t 
*cbm_len,
 return 0;
 }
 
-int psr_get_l3_cbm(struct domain *d, unsigned int socket, uint64_t *cbm)
+int psr_get_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t *cbm, enum cbm_type type)
 {
 struct psr_cat_socket_info *info = get_cat_socket_info(socket);
 
 if ( IS_ERR(info) )
 return PTR_ERR(info);
 
-*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+if ( type == PSR_CBM_TYPE_L3 )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cbm;
+else if ( type == PSR_CBM_TYPE_L3_CODE )
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.code;
+else
+*cbm = info->cos_to_cbm[d->arch.psr_cos_ids[socket]].u.cdp.data;
 
 return 0;
 }
@@ -365,10 +371,51 @@ static int write_l3_cbm(unsigned int socket, unsigned int 
cos,
 return 0;
 }
 
-int psr_set_l3_cbm(struct domain *d, unsigned int socket, uint64_t cbm)
+static int exist_same_cos(struct psr_cat_cbm *map, int cos_max,
+  uint64_t cbm_code, uint64_t cbm_data, bool_t 
cdp_enabled)
+{
+int cos;
+
+if ( !cdp_enabled )
+{
+for ( cos = 0; cos <= cos_max; cos++ )
+if ( map[cos].ref && map[cos].u.cbm == cbm_code )
+return cos;
+}
+else
+{
+for ( cos = 0; cos <= cos_max; cos++ )
+if ( map[cos].ref && map[cos].u.cdp.code == cbm_code &&
+ map[cos].u.cdp.data == cbm_data )
+return cos;
+}
+
+return -ENOENT;
+}
+
+static int pick_avail_cos(struct psr_cat_cbm *map, int cos_max, int old_cos)
+{
+int cos;
+
+/* If old cos is referred only by the domain, then use it. */
+if ( map[old_cos].ref == 1 )
+return old_cos;
+
+/* Then we pick an unused one, never pick 0 */
+for ( cos = 1; cos <= cos_max; cos++ )
+if ( map[cos].ref == 0 )
+return cos;
+
+return -EOVERFLOW;
+}
+
+int psr_set_l3_cbm(struct domain *d, unsigned int socket,
+   uint64_t cbm, enum cbm_type type)
 {
-unsigned int old_cos, cos;
-struct psr_cat_cbm *map, *found = NULL;
+