On Thu, Jan 25, 2024 at 08:22:14AM +0100, Anthony Harivel wrote:
> Starting with the "Sandy Bridge" generation, Intel CPUs provide a RAPL
> interface (Running Average Power Limit) for advertising the accumulated
> energy consumption of various power domains (e.g. CPU packages, DRAM,
> etc.).
>
> The consumption is reported via MSRs (model specific registers) like
> MSR_PKG_ENERGY_STATUS for the CPU package power domain. These MSRs are
> 64 bits registers that represent the accumulated energy consumption in
> micro Joules. They are updated by microcode every ~1ms.
>
> For now, KVM always returns 0 when the guest requests the value of
> these MSRs. Use the KVM MSR filtering mechanism to allow QEMU handle
> these MSRs dynamically in userspace.
>
> To limit the amount of system calls for every MSR call, create a new
> thread in QEMU that updates the "virtual" MSR values asynchronously.
>
> Each vCPU has its own vMSR to reflect the independence of vCPUs. The
> thread updates the vMSR values with the ratio of energy consumed of
> the whole physical CPU package the vCPU thread runs on and the
> thread's utime and stime values.
>
> All other non-vCPU threads are also taken into account. Their energy
> consumption is evenly distributed among all vCPUs threads running on
> the same physical CPU package.
>
> To overcome the problem that reading the RAPL MSR requires priviliged
> access, a socket communication between QEMU and the qemu-vmsr-helper is
> mandatory. You can specified the socket path in the parameter.
>
> This feature is activated with -accel kvm,rapl=true,path=/path/sock.sock
>
> Actual limitation:
> - Works only on Intel host CPU because AMD CPUs are using different MSR
> adresses.
>
> - Only the Package Power-Plane (MSR_PKG_ENERGY_STATUS) is reported at
> the moment.
>
> Signed-off-by: Anthony Harivel <ahari...@redhat.com>
> ---
> accel/kvm/kvm-all.c | 27 +++
> docs/specs/index.rst | 1 +
> docs/specs/rapl-msr.rst | 133 +++++++++++++
> include/sysemu/kvm_int.h | 17 ++
> target/i386/cpu.h | 8 +
> target/i386/kvm/kvm.c | 348 ++++++++++++++++++++++++++++++++++
> target/i386/kvm/meson.build | 1 +
> target/i386/kvm/vmsr_energy.c | 295 ++++++++++++++++++++++++++++
> target/i386/kvm/vmsr_energy.h | 87 +++++++++
> 9 files changed, 917 insertions(+)
> create mode 100644 docs/specs/rapl-msr.rst
> create mode 100644 target/i386/kvm/vmsr_energy.c
> create mode 100644 target/i386/kvm/vmsr_energy.h
>
> diff --git a/include/sysemu/kvm_int.h b/include/sysemu/kvm_int.h
> index 882e37e12c5b..ee2fe8817833 100644
> --- a/include/sysemu/kvm_int.h
> +++ b/include/sysemu/kvm_int.h
> @@ -14,6 +14,8 @@
> #include "qemu/accel.h"
> #include "qemu/queue.h"
> #include "sysemu/kvm.h"
> +#include "hw/boards.h"
> +#include "hw/i386/topology.h"
>
> typedef struct KVMSlot
> {
> @@ -48,6 +50,20 @@ typedef struct KVMMemoryListener {
>
> #define KVM_MSI_HASHTAB_SIZE 256
>
> +struct KVMMsrEnergy {
> + bool enable;
> + char *socket_path;
> + QemuThread msr_thr;
> + unsigned int cpus;
> + unsigned int sockets;
> + X86CPUTopoInfo topo_info;
> + const CPUArchIdList *cpu_list;
> + uint64_t *msr_value;
> + uint64_t msr_unit;
> + uint64_t msr_limit;
> + uint64_t msr_info;
> +};
> +
> enum KVMDirtyRingReaperState {
> KVM_DIRTY_RING_REAPER_NONE = 0,
> /* The reaper is sleeping */
> @@ -114,6 +130,7 @@ struct KVMState
> bool kvm_dirty_ring_with_bitmap;
> uint64_t kvm_eager_split_size; /* Eager Page Splitting chunk size */
> struct KVMDirtyRingReaper reaper;
> + struct KVMMsrEnergy msr_energy;
> NotifyVmexitOption notify_vmexit;
> uint32_t notify_window;
> uint32_t xen_version;
> @@ -2509,6 +2558,265 @@ static void register_smram_listener(Notifier *n, void
> *unused)
> &smram_address_space, 1, "kvm-smram");
> }
>
> +static void *kvm_msr_energy_thread(void *data)
> +{
> + KVMState *s = data;
> + struct KVMMsrEnergy *vmsr = &s->msr_energy;
> +
> + g_autofree package_energy_stat *pkg_stat = NULL;
> + g_autofree thread_stat *thd_stat = NULL;
> + g_autofree pid_t *thread_ids = NULL;
> + g_autofree CPUState *cpu = NULL;
> + unsigned int maxpkgs, maxcpus, maxticks;
> + g_autofree unsigned int *vpkgs_energy_stat = NULL;
> + unsigned int num_threads = 0;
> + unsigned int tmp_num_threads = 0;
> + pid_t pid;
> +
> + X86CPUTopoIDs topo_ids;
> +
> +
> + rcu_register_thread();
> +
> + /* Get QEMU PID*/
> + pid = getpid();
> +
> + /* Nb of CPUS per packages */
> + maxcpus = vmsr_get_maxcpus(0);
> +
> + /* Nb of Physical Packages on the system */
> + maxpkgs = vmsr_get_max_physical_package(maxcpus);
> +
> + /* Those MSR values should not change as well */
> + vmsr->msr_unit = vmsr_read_msr(MSR_RAPL_POWER_UNIT, 0, pid,
> + s->msr_energy.socket_path);
> + vmsr->msr_limit = vmsr_read_msr(MSR_PKG_POWER_LIMIT, 0, pid,
> + s->msr_energy.socket_path);
> + vmsr->msr_info = vmsr_read_msr(MSR_PKG_POWER_INFO, 0, pid,
> + s->msr_energy.socket_path);
> +
> + /* Allocate memory for each package energy status */
> + pkg_stat = (package_energy_stat *)
> + g_new0(package_energy_stat, maxpkgs);
> +
> + /* Pre-allocate memory for thread stats */
> + thd_stat = g_new0(thread_stat, 1);
> +
> + /* Pre-allocate memory for holding Virtual Package Energy counter */
> + vpkgs_energy_stat = g_new0(unsigned int, vmsr->sockets);
> +
> + /*
> + * Max numbers of ticks per package
> + * time in second * number of ticks/second * Number of cores / package
> + * ex: for 100 ticks/second/CPU, 12 CPUs per Package gives 1200 ticks max
> + */
> + maxticks = (MSR_ENERGY_THREAD_SLEEP_US / 1000000)
> + * sysconf(_SC_CLK_TCK) * maxcpus;
> +
> + while (true) {
> + /* Get all qemu threads id */
> + thread_ids = vmsr_get_thread_ids(pid, &num_threads);
This is allolcating thread_ids
> +
> + if (thread_ids == NULL) {
> + goto clean;
> + }
> +
> + if (tmp_num_threads < num_threads) {
> + thd_stat = g_renew(thread_stat, thd_stat, num_threads);
> + }
> +
> + tmp_num_threads = num_threads;
> +
> + /* Populate all the thread stats */
> + for (int i = 0; i < num_threads; i++) {
> + thd_stat[i].utime = g_new0(unsigned long long, 2);
> + thd_stat[i].stime = g_new0(unsigned long long, 2);
> + thd_stat[i].thread_id = thread_ids[i];
> + vmsr_read_thread_stat(&thd_stat[i], pid, 0);
> + thd_stat[i].numa_node_id = numa_node_of_cpu(thd_stat[i].cpu_id);
> + }
This loop is allocating memory for utime & stime..
> +
> + /* Zero out the memory */
> + for (int i = 0; i < num_threads; i++) {
> + memset(thd_stat[i].utime, 0, 2 * sizeof(unsigned long long));
> + memset(thd_stat[i].stime, 0, 2 * sizeof(unsigned long long));
> + }
> + memset(thd_stat, 0, num_threads * sizeof(thread_stat));
..and here we're just wiping the thd_stat pointer, so this is surey a
memory leak of the stime & utime fields.
> + memset(thread_ids, 0, sizeof(pid_t));
Nothing is free'ing thread_ids which was allocated earlier in the
loop.
> + }
> +
> +clean:
> + rcu_unregister_thread();
> + return NULL;
> +}
> +
> +static int kvm_msr_energy_thread_init(KVMState *s, MachineState *ms)
> +{
> + struct KVMMsrEnergy *r = &s->msr_energy;
> +
> + vmsr_init_topo_info(&r->topo_info, ms);
> +
> + /* Retrieve the number of vCPU */
> + r->cpus = ms->smp.cpus;
> +
> + /* Retrieve the number of sockets */
> + r->sockets = ms->smp.sockets;
> +
> + /* Allocate register memory (MSR_PKG_STATUS) for each vCPU */
> + r->msr_value = g_new0(uint64_t, r->cpus);
> +
> + /* Retrieve the CPUArchIDlist */
> + r->x86_cpu_list = x86_possible_cpu_arch_ids(ms);
> +
> + qemu_thread_create(&r->msr_thr, "kvm-msr",
> + kvm_msr_energy_thread,
> + s, QEMU_THREAD_JOINABLE);
> +
> + return 0;
> +}
> +
> int kvm_arch_get_default_type(MachineState *ms)
> {
> return 0;
> @@ -2711,6 +3019,46 @@ int kvm_arch_init(MachineState *ms, KVMState *s)
> strerror(-ret));
> exit(1);
> }
> +
> + if (s->msr_energy.enable == true) {
This looks to be where we need to check that both the host CPU
vendor is intel, and the guest CPU vendor is intel, and that
the host CPU has the RAPL feature we're using.
> +
> + r = kvm_filter_msr(s, MSR_RAPL_POWER_UNIT,
> + kvm_rdmsr_rapl_power_unit, NULL);
> + if (!r) {
> + error_report("Could not install MSR_RAPL_POWER_UNIT \
> + handler: %s",
> + strerror(-ret));
> + exit(1);
> + }
> +
> + r = kvm_filter_msr(s, MSR_PKG_POWER_LIMIT,
> + kvm_rdmsr_pkg_power_limit, NULL);
> + if (!r) {
> + error_report("Could not install MSR_PKG_POWER_LIMIT \
> + handler: %s",
> + strerror(-ret));
> + exit(1);
> + }
> +
> + r = kvm_filter_msr(s, MSR_PKG_POWER_INFO,
> + kvm_rdmsr_pkg_power_info, NULL);
> + if (!r) {
> + error_report("Could not install MSR_PKG_POWER_INFO \
> + handler: %s",
> + strerror(-ret));
> + exit(1);
> + }
> + r = kvm_filter_msr(s, MSR_PKG_ENERGY_STATUS,
> + kvm_rdmsr_pkg_energy_status, NULL);
> + if (!r) {
> + error_report("Could not install MSR_PKG_ENERGY_STATUS \
> + handler: %s",
> + strerror(-ret));
> + exit(1);
> + } else {
> + kvm_msr_energy_thread_init(s, ms);
> + }
> + }
> }
>
> return 0;
With regards,
Daniel
--
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org -o- https://fstop138.berrange.com :|
|: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
