Add Memory Bandwidth Monitoring(MBM) for VMs. Two types of monitoring are supported: total and local memory bandwidth monitoring. To use it, CMT should be enabled in hypervisor.
Signed-off-by: Chao Peng <chao.p.p...@linux.intel.com> --- Changes in v9: 1. Refactor code in xc_psr_cmt_get_data. 2. Move bandwidth calculation(sleep) from libxl to xl. 3. Broadcast feature with LIBXL_HAVE_PSR_MBM. 4. Check event mask with libxl_psr_cmt_type_supported. 5. Coding style/Document fix. Changes in v6: 1. Remove DISABLE_IRQ flag as hypervisor disable IRQ for MSR_IA32_TSC implicitly. Changes in v5: 1. Add MBM description in xen command line. 2. Use the tsc from hypervisor directly which is already ns. 3. Call resource_op with DISABLE_IRQ flag. Changes in v4: 1. Get timestamp from hypervisor and use that for bandwidth calculation. 2. Minor document and coding style fix. --- docs/man/xl.pod.1 | 11 +++++- docs/misc/xen-command-line.markdown | 3 ++ tools/libxc/include/xenctrl.h | 6 +++- tools/libxc/xc_msr_x86.h | 1 + tools/libxc/xc_psr.c | 68 ++++++++++++++++++++++++++++++------- tools/libxl/libxl.h | 17 ++++++++++ tools/libxl/libxl_psr.c | 44 ++++++++++++++++++++++-- tools/libxl/libxl_types.idl | 2 ++ tools/libxl/xl_cmdimpl.c | 67 ++++++++++++++++++++++++++++++++++++ tools/libxl/xl_cmdtable.c | 4 ++- 10 files changed, 204 insertions(+), 19 deletions(-) diff --git a/docs/man/xl.pod.1 b/docs/man/xl.pod.1 index 6b89ba8..cd80ffc 100644 --- a/docs/man/xl.pod.1 +++ b/docs/man/xl.pod.1 @@ -1461,6 +1461,13 @@ is domain level. To monitor a specific domain, just attach the domain id with the monitoring service. When the domain doesn't need to be monitored any more, detach the domain id from the monitoring service. +Intel Broadwell and later server platforms also offer total/local memory +bandwidth monitoring. Xen supports per-domain monitoring for these two +additional monitoring types. Both memory bandwidth monitoring and L3 cache +occupancy monitoring share the same set of underlying monitoring service. Once +a domain is attached to the monitoring service, monitoring data can be showed +for any of these monitoring types. + =over 4 =item B<psr-cmt-attach> [I<domain-id>] @@ -1475,7 +1482,9 @@ detach: Detach the platform shared resource monitoring service from a domain. Show monitoring data for a certain domain or all domains. Current supported monitor types are: - - "cache-occupancy": showing the L3 cache occupancy. + - "cache-occupancy": showing the L3 cache occupancy(KB). + - "total-mem-bandwidth": showing the total memory bandwidth(KB/s). + - "local-mem-bandwidth": showing the local memory bandwidth(KB/s). =back diff --git a/docs/misc/xen-command-line.markdown b/docs/misc/xen-command-line.markdown index a061aa4..0491dbb 100644 --- a/docs/misc/xen-command-line.markdown +++ b/docs/misc/xen-command-line.markdown @@ -1097,6 +1097,9 @@ The following resources are available: L3 cache occupancy. * `cmt` instructs Xen to enable/disable Cache Monitoring Technology. * `rmid_max` indicates the max value for rmid. +* Memory Bandwidth Monitoring (Broadwell and later). Information regarding the + total/local memory bandwidth. Follow the same options with Cache Monitoring + Technology. ### reboot > `= t[riple] | k[bd] | a[cpi] | p[ci] | n[o] [, [w]arm | [c]old]` diff --git a/tools/libxc/include/xenctrl.h b/tools/libxc/include/xenctrl.h index c62f371..8f964ad 100644 --- a/tools/libxc/include/xenctrl.h +++ b/tools/libxc/include/xenctrl.h @@ -2688,6 +2688,8 @@ int xc_resource_op(xc_interface *xch, uint32_t nr_ops, xc_resource_op_t *ops); #if defined(__i386__) || defined(__x86_64__) enum xc_psr_cmt_type { XC_PSR_CMT_L3_OCCUPANCY, + XC_PSR_CMT_TOTAL_MEM_BANDWIDTH, + XC_PSR_CMT_LOCAL_MEM_BANDWIDTH, }; typedef enum xc_psr_cmt_type xc_psr_cmt_type; int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid); @@ -2697,10 +2699,12 @@ int xc_psr_cmt_get_domain_rmid(xc_interface *xch, uint32_t domid, int xc_psr_cmt_get_total_rmid(xc_interface *xch, uint32_t *total_rmid); int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch, uint32_t *upscaling_factor); +int xc_psr_cmt_get_l3_event_mask(xc_interface *xch, uint32_t *event_mask); int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu, uint32_t *l3_cache_size); int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu, - uint32_t psr_cmt_type, uint64_t *monitor_data); + uint32_t psr_cmt_type, uint64_t *monitor_data, + uint64_t *tsc); int xc_psr_cmt_enabled(xc_interface *xch); #endif diff --git a/tools/libxc/xc_msr_x86.h b/tools/libxc/xc_msr_x86.h index 7c3e1a3..7f100e7 100644 --- a/tools/libxc/xc_msr_x86.h +++ b/tools/libxc/xc_msr_x86.h @@ -20,6 +20,7 @@ #ifndef XC_MSR_X86_H #define XC_MSR_X86_H +#define MSR_IA32_TSC 0x00000010 #define MSR_IA32_CMT_EVTSEL 0x00000c8d #define MSR_IA32_CMT_CTR 0x00000c8e diff --git a/tools/libxc/xc_psr.c b/tools/libxc/xc_psr.c index cfae172..5482c8e 100644 --- a/tools/libxc/xc_psr.c +++ b/tools/libxc/xc_psr.c @@ -23,6 +23,8 @@ #define IA32_CMT_CTR_ERROR_MASK (0x3ull << 62) #define EVTID_L3_OCCUPANCY 0x1 +#define EVTID_TOTAL_MEM_BANDWIDTH 0x2 +#define EVTID_LOCAL_MEM_BANDWIDTH 0x3 int xc_psr_cmt_attach(xc_interface *xch, uint32_t domid) { @@ -112,6 +114,23 @@ int xc_psr_cmt_get_l3_upscaling_factor(xc_interface *xch, return rc; } +int xc_psr_cmt_get_l3_event_mask(xc_interface *xch, uint32_t *event_mask) +{ + int rc; + DECLARE_SYSCTL; + + sysctl.cmd = XEN_SYSCTL_psr_cmt_op; + sysctl.u.psr_cmt_op.cmd = + XEN_SYSCTL_PSR_CMT_get_l3_event_mask; + sysctl.u.psr_cmt_op.flags = 0; + + rc = xc_sysctl(xch, &sysctl); + if ( !rc ) + *event_mask = sysctl.u.psr_cmt_op.u.data; + + return rc; +} + int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu, uint32_t *l3_cache_size) { @@ -139,11 +158,13 @@ int xc_psr_cmt_get_l3_cache_size(xc_interface *xch, uint32_t cpu, } int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu, - xc_psr_cmt_type type, uint64_t *monitor_data) + xc_psr_cmt_type type, uint64_t *monitor_data, + uint64_t *tsc) { xc_resource_op_t op; - xc_resource_entry_t entries[2]; - uint32_t evtid; + xc_resource_entry_t entries[3]; + xc_resource_entry_t *tsc_entry = NULL; + uint32_t evtid, nr = 0; int rc; switch ( type ) @@ -151,33 +172,54 @@ int xc_psr_cmt_get_data(xc_interface *xch, uint32_t rmid, uint32_t cpu, case XC_PSR_CMT_L3_OCCUPANCY: evtid = EVTID_L3_OCCUPANCY; break; + case XC_PSR_CMT_TOTAL_MEM_BANDWIDTH: + evtid = EVTID_TOTAL_MEM_BANDWIDTH; + break; + case XC_PSR_CMT_LOCAL_MEM_BANDWIDTH: + evtid = EVTID_LOCAL_MEM_BANDWIDTH; + break; default: return -1; } - entries[0].u.cmd = XEN_RESOURCE_OP_MSR_WRITE; - entries[0].idx = MSR_IA32_CMT_EVTSEL; - entries[0].val = (uint64_t)rmid << 32 | evtid; - entries[0].rsvd = 0; + entries[nr].u.cmd = XEN_RESOURCE_OP_MSR_WRITE; + entries[nr].idx = MSR_IA32_CMT_EVTSEL; + entries[nr].val = (uint64_t)rmid << 32 | evtid; + entries[nr].rsvd = 0; + nr++; - entries[1].u.cmd = XEN_RESOURCE_OP_MSR_READ; - entries[1].idx = MSR_IA32_CMT_CTR; - entries[1].val = 0; - entries[1].rsvd = 0; + entries[nr].u.cmd = XEN_RESOURCE_OP_MSR_READ; + entries[nr].idx = MSR_IA32_CMT_CTR; + entries[nr].val = 0; + entries[nr].rsvd = 0; + nr++; + + if ( tsc != NULL ) + { + tsc_entry = &entries[nr]; + entries[nr].u.cmd = XEN_RESOURCE_OP_MSR_READ; + entries[nr].idx = MSR_IA32_TSC; + entries[nr].val = 0; + entries[nr].rsvd = 0; + nr++; + } op.cpu = cpu; - op.nr_entries = 2; + op.nr_entries = nr; op.entries = entries; rc = xc_resource_op(xch, 1, &op); if ( rc < 0 ) return rc; - if ( op.result !=2 || entries[1].val & IA32_CMT_CTR_ERROR_MASK ) + if ( op.result != nr || entries[1].val & IA32_CMT_CTR_ERROR_MASK ) return -1; *monitor_data = entries[1].val; + if ( tsc_entry != NULL ) + *tsc = tsc_entry->val; + return 0; } diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index 28b6325..9b04927 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -690,6 +690,13 @@ void libxl_mac_copy(libxl_ctx *ctx, libxl_mac *dst, libxl_mac *src); * If this is defined, the Cache Monitoring Technology feature is supported. */ #define LIBXL_HAVE_PSR_CMT 1 + +/* + * LIBXL_HAVE_PSR_MBM + * + * If this is defined, the Memory Bandwidth Monitoring feature is supported. + */ +#define LIBXL_HAVE_PSR_MBM 1 #endif typedef char **libxl_string_list; @@ -1463,6 +1470,16 @@ int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx, uint32_t *l3_cache_occupancy); #endif +#ifdef LIBXL_HAVE_PSR_MBM +int libxl_psr_cmt_type_supported(libxl_ctx *ctx, libxl_psr_cmt_type type); +int libxl_psr_cmt_get_mem_bandwidth_sample(libxl_ctx *ctx, + uint32_t domid, + uint32_t socketid, + bool total, + uint64_t *sample_r, + uint64_t *tsc_r); +#endif + /* misc */ /* Each of these sets or clears the flag according to whether the diff --git a/tools/libxl/libxl_psr.c b/tools/libxl/libxl_psr.c index 1e1f7ee..ed444e2 100644 --- a/tools/libxl/libxl_psr.c +++ b/tools/libxl/libxl_psr.c @@ -161,11 +161,30 @@ out: return rc; } +int libxl_psr_cmt_type_supported(libxl_ctx *ctx, libxl_psr_cmt_type type) +{ + GC_INIT(ctx); + uint32_t event_mask; + int rc; + + rc = xc_psr_cmt_get_l3_event_mask(ctx->xch, &event_mask); + if (rc < 0) { + libxl__psr_cmt_log_err_msg(gc, errno); + rc = 0; + } else { + rc = event_mask & (1 << (type - 1)); + } + + GC_FREE; + return rc; +} + static int libxl__psr_cmt_get_l3_monitoring_data(libxl__gc *gc, uint32_t domid, xc_psr_cmt_type type, uint32_t socketid, - uint64_t *data_r) + uint64_t *data_r, + uint64_t *tsc_r) { unsigned int rmid; uint32_t upscaling_factor; @@ -187,7 +206,7 @@ static int libxl__psr_cmt_get_l3_monitoring_data(libxl__gc *gc, goto out; } - rc = xc_psr_cmt_get_data(CTX->xch, rmid, cpu, type, &monitor_data); + rc = xc_psr_cmt_get_data(CTX->xch, rmid, cpu, type, &monitor_data, tsc_r); if (rc < 0) { LOGE(ERROR, "failed to get monitoring data"); rc = ERROR_FAIL; @@ -218,7 +237,7 @@ int libxl_psr_cmt_get_cache_occupancy(libxl_ctx *ctx, rc = libxl__psr_cmt_get_l3_monitoring_data(gc, domid, XC_PSR_CMT_L3_OCCUPANCY, - socketid, &data); + socketid, &data, NULL); if (rc < 0) goto out; @@ -229,6 +248,25 @@ out: return rc; } +int libxl_psr_cmt_get_mem_bandwidth_sample(libxl_ctx *ctx, + uint32_t domid, + uint32_t socketid, + bool total, + uint64_t *sample_r, + uint64_t *tsc_r) +{ + GC_INIT(ctx); + int rc; + xc_psr_cmt_type type = total ? XC_PSR_CMT_TOTAL_MEM_BANDWIDTH + : XC_PSR_CMT_LOCAL_MEM_BANDWIDTH; + + rc = libxl__psr_cmt_get_l3_monitoring_data(gc, domid, type, socketid, + sample_r, tsc_r); + + GC_FREE; + return rc; +} + /* * Local variables: * mode: C diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 1214d2e..46d160e 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -694,4 +694,6 @@ libxl_event = Struct("event",[ libxl_psr_cmt_type = Enumeration("psr_cmt_type", [ (1, "CACHE_OCCUPANCY"), + (2, "TOTAL_MEM_BANDWIDTH"), + (3, "LOCAL_MEM_BANDWIDTH"), ]) diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index a0b50e7..3378e64 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -7807,6 +7807,54 @@ out: } #ifdef LIBXL_HAVE_PSR_CMT + +#define MBM_SAMPLE_RETRY_MAX 4 +static int psr_cmt_get_mem_bandwidth(uint32_t domid, + uint32_t socketid, + bool total, + uint64_t *bandwidth_r) +{ + uint64_t sample1, sample2; + uint64_t tsc1, tsc2; + int retry_attempts = 0; + int rc; + + while (1) { + rc = libxl_psr_cmt_get_mem_bandwidth_sample(ctx, domid, socketid, + total, &sample1, &tsc1); + if (rc < 0) + return rc; + + usleep(10000); + + rc = libxl_psr_cmt_get_mem_bandwidth_sample(ctx, domid, socketid, + total, &sample2, &tsc2); + if (rc < 0) + return rc; + + if (tsc2 <= tsc1) + return -1; + + /* + * Hardware guarantees at most 1 overflow can happen if the duration + * between two samples is less than 1 second. Note that tsc returned + * from hypervisor is already-scaled time(ns). + */ + if (tsc2 - tsc1 < 1000000000 && sample2 >= sample1) + break; + + if (retry_attempts < MBM_SAMPLE_RETRY_MAX) { + retry_attempts++; + } else { + fprintf(stderr, "event counter overflowed\n"); + return -1; + } + } + + *bandwidth_r = (sample2 - sample1) * 1000000000 / (tsc2 - tsc1) / 1024; + return 0; +} + static void psr_cmt_print_domain_l3_info(libxl_dominfo *dominfo, libxl_psr_cmt_type type, uint32_t nr_sockets) @@ -7814,6 +7862,7 @@ static void psr_cmt_print_domain_l3_info(libxl_dominfo *dominfo, char *domain_name; uint32_t socketid; uint32_t l3_cache_occupancy; + uint64_t mem_bandwidth; if (!libxl_psr_cmt_domain_attached(ctx, dominfo->domid)) return; @@ -7831,6 +7880,16 @@ static void psr_cmt_print_domain_l3_info(libxl_dominfo *dominfo, &l3_cache_occupancy)) printf("%13u KB", l3_cache_occupancy); break; + case LIBXL_PSR_CMT_TYPE_TOTAL_MEM_BANDWIDTH: + if (!psr_cmt_get_mem_bandwidth(dominfo->domid, socketid, true, + &mem_bandwidth)) + printf("%11"PRIu64" KB/s", mem_bandwidth); + break; + case LIBXL_PSR_CMT_TYPE_LOCAL_MEM_BANDWIDTH: + if (!psr_cmt_get_mem_bandwidth(dominfo->domid, socketid, false, + &mem_bandwidth)) + printf("%11"PRIu64" KB/s", mem_bandwidth); + break; default: return; } @@ -7851,6 +7910,12 @@ static int psr_cmt_show_l3_info(libxl_psr_cmt_type type, uint32_t domid) return -1; } + if (!libxl_psr_cmt_type_supported(ctx, type)) { + fprintf(stderr, "Monitor type '%s' is not supported in the system\n", + libxl_psr_cmt_type_to_string(type)); + return -1; + } + libxl_physinfo_init(&info); rc = libxl_get_physinfo(ctx, &info); if (rc < 0) { @@ -7968,6 +8033,8 @@ int main_psr_cmt_show(int argc, char **argv) switch (type) { case LIBXL_PSR_CMT_TYPE_CACHE_OCCUPANCY: + case LIBXL_PSR_CMT_TYPE_TOTAL_MEM_BANDWIDTH: + case LIBXL_PSR_CMT_TYPE_LOCAL_MEM_BANDWIDTH: ret = psr_cmt_show_l3_info(type, domid); break; default: diff --git a/tools/libxl/xl_cmdtable.c b/tools/libxl/xl_cmdtable.c index 4b30d3d..22ab63b 100644 --- a/tools/libxl/xl_cmdtable.c +++ b/tools/libxl/xl_cmdtable.c @@ -538,7 +538,9 @@ struct cmd_spec cmd_table[] = { "Show Cache Monitoring Technology information", "<PSR-CMT-Type> <Domain>", "Available monitor types:\n" - "\"cache_occupancy\": Show L3 cache occupancy\n", + "\"cache_occupancy\": Show L3 cache occupancy(KB)\n" + "\"total_mem_bandwidth\": Show total memory bandwidth(KB/s)\n" + "\"local_mem_bandwidth\": Show local memory bandwidth(KB/s)\n", }, #endif }; -- 1.9.1 _______________________________________________ Xen-devel mailing list Xen-devel@lists.xen.org http://lists.xen.org/xen-devel