From: Kyle Hale Date: Tue, 18 Jun 2013 22:16:38 +0000 (-0500) Subject: Add interface for power monitoring and telemetry option for power-related statistics X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=ec75bbb093fa99a4be794f0f564867c0091ccb85 Add interface for power monitoring and telemetry option for power-related statistics I added telemetry output for Intel's RAPL power monitoring capability (present since Sandy Bridge). This code is pretty heavily based on the PMU codebase, so it might make sense to merge them together at some point, but for now the semantics of PMU and RAPL seemed different enough to justify having them separate. The code is organized in a way such that if AMD adds similar support or if Intel adds new RAPL counters, they should be fairly straightforward to integrate. --- diff --git a/Kconfig b/Kconfig index cfbabb2..7e231b0 100644 --- a/Kconfig +++ b/Kconfig @@ -165,7 +165,13 @@ config PMU_TELEMETRY Enable telemetry information for a range of PMU counters This causes the currently configured PMU counts to be printed - +config PWRSTAT_TELEMETRY + bool "Enable power statistics telemetry" + default n + depends on TELEMETRY && HOST_PWRSTAT + help + Enable telemetry information for power/energy counters + config EXPERIMENTAL bool "Enable Experimental options" diff --git a/linux_module/Makefile b/linux_module/Makefile index 99823e1..6824eea 100644 --- a/linux_module/Makefile +++ b/linux_module/Makefile @@ -35,6 +35,7 @@ v3vee-$(V3_CONFIG_GRAPHICS_CONSOLE) += iface-graphics-console.o v3vee-$(V3_CONFIG_EXT_MACH_CHECK) += mcheck.o v3vee-$(V3_CONFIG_HOST_PMU) += iface-pmu.o +v3vee-$(V3_CONFIG_HOST_PWRSTAT) += iface-pwrstat.o v3vee-$(V3_CONFIG_VNET) += palacios-vnet.o \ palacios-vnet-ctrl.o \ diff --git a/linux_module/iface-pwrstat.c b/linux_module/iface-pwrstat.c new file mode 100644 index 0000000..2ae1f54 --- /dev/null +++ b/linux_module/iface-pwrstat.c @@ -0,0 +1,316 @@ +/* + * + * Intel RAPL (Sandy Bridge and above) Accessor + * (c) Kyle C. Hale, Chang Bae 2013 + * + * + */ +#include +#include +#include + +#include +#include + +#include + +#include "vm.h" +#include "palacios.h" +#include "util-queue.h" +#include "linux-exts.h" +#include "iface-pwrstat.h" + + +enum rapl_domain_id { + RAPL_DOMAIN_PKG, + RAPL_DOMAIN_PP0, + RAPL_DOMAIN_PP1, + RAPL_DOMAIN_DRAM, + RAPL_DOMAIN_MAX +}; + + +struct rapl_domain_msr { + int limit; + int status; +}; + + +struct rapl_domain { + enum rapl_domain_id domain_id; + struct rapl_domain_msr msrs; + int valid; +}; + + +static struct rapl_domain rapl_domains[] = { + [RAPL_DOMAIN_PKG] = { + .domain_id = RAPL_DOMAIN_PKG, + .msrs = { + .limit = MSR_PKG_RAPL_POWER_LIMIT, + .status = MSR_PKG_ENERGY_STATUS, + }, + .valid = 1, + }, + [RAPL_DOMAIN_PP0] = { + .domain_id = RAPL_DOMAIN_PP0, + .msrs = { + .limit = MSR_PP0_POWER_LIMIT, + .status = MSR_PP0_ENERGY_STATUS, + }, + .valid = 1, + }, + [RAPL_DOMAIN_PP1] = { + .domain_id = RAPL_DOMAIN_PP1, + .msrs = { + .limit = MSR_PP1_POWER_LIMIT, + .status = MSR_PP1_ENERGY_STATUS, + }, + }, + [RAPL_DOMAIN_DRAM] = { + .domain_id = RAPL_DOMAIN_DRAM, + .msrs = { + .limit = MSR_DRAM_POWER_LIMIT, + .status = MSR_DRAM_ENERGY_STATUS, + }, + }, +}; + +static unsigned int power_unit_divisor; +static unsigned int energy_unit_divisor; +static unsigned int time_unit_divisor; + +enum unit_type { + POWER_UNIT, + ENERGY_UNIT, + TIME_UNIT +}; + +static inline void cpuid_string(u32 id, u32 dest[4]) { + asm volatile("cpuid" + :"=a"(*dest),"=b"(*(dest+1)),"=c"(*(dest+2)),"=d"(*(dest+3)) + :"a"(id)); +} + + +static int rapl_check_unit (void) +{ + u64 output; + u32 value; + + rdmsrl(MSR_RAPL_POWER_UNIT, output); + + /* energy unit: 1/enery_unit_divisor Joules */ + value = (output & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET; + energy_unit_divisor = 1 << value; + + /* power unit: 1/power_unit_divisor Watts */ + value = (output & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET; + power_unit_divisor = 1 << value; + + /* time unit: 1/time_unit_divisor Seconds */ + value =(output & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET; + time_unit_divisor = 1 << value; + + return 0; +} + + +static u64 rapl_unit_xlate(enum unit_type type, u64 value, int action) +{ + u64 divisor; + + switch (type) { + case POWER_UNIT: + divisor = power_unit_divisor; + break; + case ENERGY_UNIT: + divisor = energy_unit_divisor; + break; + case TIME_UNIT: + divisor = time_unit_divisor; + break; + default: + return 0; + }; + + if (action) + return value * divisor; /* value is from users */ + else + return div64_u64(value, divisor); /* value is from MSR */ +} + + +static int rapl_read_energy(struct rapl_domain * domain) +{ + u64 value; + u32 msr = domain->msrs.status; + + rdmsrl(msr, value); + return rapl_unit_xlate(ENERGY_UNIT, value, 0); +} + + +static int rapl_ctr_valid (v3_pwrstat_ctr_t ctr) +{ + switch (ctr) { + case V3_PWRSTAT_PKG_ENERGY: + return rapl_domains[RAPL_DOMAIN_PKG].valid; + case V3_PWRSTAT_CORE_ENERGY: + return rapl_domains[RAPL_DOMAIN_PP0].valid; + case V3_PWRSTAT_EXT_ENERGY: + return rapl_domains[RAPL_DOMAIN_PP1].valid; + case V3_PWRSTAT_DRAM_ENERGY: + return rapl_domains[RAPL_DOMAIN_DRAM].valid; + default: + return 0; + } + + return 0; +} + + +static uint64_t rapl_get_value (v3_pwrstat_ctr_t ctr) +{ + switch (ctr) { + case V3_PWRSTAT_PKG_ENERGY: + return rapl_read_energy(&rapl_domains[RAPL_DOMAIN_PKG]); + case V3_PWRSTAT_CORE_ENERGY: + return rapl_read_energy(&rapl_domains[RAPL_DOMAIN_PP0]); + case V3_PWRSTAT_EXT_ENERGY: + return rapl_read_energy(&rapl_domains[RAPL_DOMAIN_PP1]); + case V3_PWRSTAT_DRAM_ENERGY: + return rapl_read_energy(&rapl_domains[RAPL_DOMAIN_DRAM]); + default: + ERROR("Invalid Powerstat Counter\n"); + return -1; + } + + return -1; +} + + +static int get_cpu_vendor (char name[13]) +{ + u32 dest[4]; + u32 maxid; + + cpuid_string(0,dest); + maxid=dest[0]; + ((u32*)name)[0]=dest[1]; + ((u32*)name)[1]=dest[3]; + ((u32*)name)[2]=dest[2]; + name[12]=0; + + return maxid; +} + + +static int get_cpu_model (void) +{ + u32 dest[4] = {0, 0, 0, 0}; + u32 ex_model, model; + cpuid_string(1, dest); + ex_model = (dest[0] & 0xF0000U) >> 16; + model = (dest[0] & 0xF0U) >> 4; + return (model + (ex_model << 4)); +} + + +static int supports_rapl (void) +{ + unsigned int model = get_cpu_model(); + switch (model) { + case SANDY_BRIDGE_E3_MODEL_NO: + return 1; + case SANDY_BRIDGE_E5_MODEL_NO: + return 1; + case IVY_BRIDGE_MODEL_NO: + return 1; + case HASWELL_MODEL_NO: + return 1; + default: + break; + } + + return 0; +} + + +static int is_intel(void) +{ + char name[13]; + get_cpu_vendor(name); + return !strcmp(name,"GenuineIntel"); +} + + +static int rapl_init (void) +{ + if (supports_rapl()) { + INFO("Intel RAPL featureset detected\n"); + } else { + ERROR("This machine does not support Intel RAPL functionality\n"); + return -1; + } + + if (get_cpu_model() == SANDY_BRIDGE_E3_MODEL_NO) { + INFO("Sandy Bridge E3, supports PP1 (no DRAM)\n"); + rapl_domains[RAPL_DOMAIN_PP1].valid = 1; + } else if (get_cpu_model() == SANDY_BRIDGE_E5_MODEL_NO) { + INFO("Sandy Bridge E5, supports DRAM domain\n"); + rapl_domains[RAPL_DOMAIN_DRAM].valid = 1; + } + + if (rapl_check_unit()) { + return -1; + } + + return 0; +} + + +static int rapl_deinit (void) +{ + INFO("Intel RAPL deinit\n"); + return 0; +} + + +static struct v3_pwrstat_iface intel_rapl_pwrstat = { + .init = rapl_init, + .deinit = rapl_deinit, + .ctr_valid = rapl_ctr_valid, + .get_value = rapl_get_value, +}; + + +static int pwrstat_init (void) +{ + if (is_intel()) { + if (supports_rapl()) { + INFO("Palacios Power stats using Intel RAPL\n"); + V3_Init_Pwrstat(&intel_rapl_pwrstat); + } else { + INFO("Intel machine, but no RAPL functionality, so no power statistics available\n"); + return -1; + } + } else { + ERROR("Not an Intel Machine, no power statistics available\n"); + return -1; + } + + return 0; +} + +/* if AMD comes up with something it can go here */ + +static struct linux_ext pwrstat_ext = { + .name = "POWERSTAT", + .init = pwrstat_init, + .deinit = NULL, + .guest_init = NULL, + .guest_deinit = NULL +}; + +register_extension(&pwrstat_ext); diff --git a/linux_module/iface-pwrstat.h b/linux_module/iface-pwrstat.h new file mode 100644 index 0000000..6d64cc1 --- /dev/null +++ b/linux_module/iface-pwrstat.h @@ -0,0 +1,46 @@ +#ifndef __PALACIOS_PWRSTAT_H__ +#define __PALACIOS_PWRSTAT_H__ + +/* THESE ARE INTEL SPECIFIC (Sandy Bridge and up) */ + +#define SANDY_BRIDGE_E3_MODEL_NO 0x2A +#define SANDY_BRIDGE_E5_MODEL_NO 0x2D +#define IVY_BRIDGE_MODEL_NO 0x3A +/* WARNING WARNING: this is speculation... */ +#define HASWELL_MODEL_NO 0x4A + +#define MSR_RAPL_POWER_UNIT 0x606 + +#define MSR_PKG_RAPL_POWER_LIMIT 0x610 +#define MSR_PKG_ENERGY_STATUS 0x611 +#define MSR_PKG_PERF_STATUS 0x613 +#define MSR_PKG_POWER_INFO 0x614 + +/* PP0 RAPL Domain */ +#define MSR_PP0_POWER_LIMIT 0x638 +#define MSR_PP0_ENERGY_STATUS 0x639 +#define MSR_PP0_POLICY 0x63A +#define MSR_PP0_PERF_STATUS 0x63B + +/* PP1 RAPL Domain, may reflect to uncore devices */ +#define MSR_PP1_POWER_LIMIT 0x640 +#define MSR_PP1_ENERGY_STATUS 0x641 +#define MSR_PP1_POLICY 0x642 + +/* DRAM RAPL Domain */ +#define MSR_DRAM_POWER_LIMIT 0x618 +#define MSR_DRAM_ENERGY_STATUS 0x619 +#define MSR_DRAM_PERF_STATUS 0x61B +#define MSR_DRAM_POWER_INFO 0x61C + +/* RAPL UNIT BITMASK */ +#define POWER_UNIT_OFFSET 0 +#define POWER_UNIT_MASK 0x0F + +#define ENERGY_UNIT_OFFSET 0x08 +#define ENERGY_UNIT_MASK 0x1F00 + +#define TIME_UNIT_OFFSET 0x10 +#define TIME_UNIT_MASK 0xF000 + +#endif diff --git a/palacios/include/interfaces/vmm_pwrstat.h b/palacios/include/interfaces/vmm_pwrstat.h new file mode 100644 index 0000000..694ca5a --- /dev/null +++ b/palacios/include/interfaces/vmm_pwrstat.h @@ -0,0 +1,50 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2013, Kyle C. Hale + * Copyright (c) 2013, The V3VEE Project + * All rights reserved. + * + * Author: Kyle C. Hale + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#ifndef __RAPL_H__ +#define __RAPL_H__ + +#include + +typedef enum { + V3_PWRSTAT_PKG_ENERGY, + V3_PWRSTAT_CORE_ENERGY, + V3_PWRSTAT_EXT_ENERGY, /* "Power plane 1", e.g. graphics peripheral */ + V3_PWRSTAT_DRAM_ENERGY, +} v3_pwrstat_ctr_t; + +/* note that (on Intel at least) RAPL doesn't adhere to PMU semantics */ +struct v3_pwrstat_iface { + int (*init)(void); + int (*deinit)(void); + int (*ctr_valid)(v3_pwrstat_ctr_t ctr); + uint64_t (*get_value)(v3_pwrstat_ctr_t ctr); +}; + +extern void V3_Init_Pwrstat (struct v3_pwrstat_iface * palacios_pwrstat); + +#ifdef __V3VEE__ +void v3_pwrstat_init(void); +int v3_pwrstat_ctr_valid(v3_pwrstat_ctr_t ctr); +uint64_t v3_pwrstat_get_value(v3_pwrstat_ctr_t ctr); +void v3_pwrstat_deinit(void); + +#endif + +#endif diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 1e999ba..3f05a99 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -53,6 +53,10 @@ #include #endif +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY +#include +#endif + #ifdef V3_CONFIG_SYMBIOTIC #include struct v3_sym_core_state; @@ -126,6 +130,10 @@ struct guest_info { struct v3_core_pmu_telemetry pmu_telem; #endif +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + struct v3_core_pwrstat_telemetry pwrstat_telem; +#endif + /* struct v3_core_dev_mgr core_dev_mgr; */ void * decoder_state; diff --git a/palacios/include/palacios/vmm_pwrstat_telemetry.h b/palacios/include/palacios/vmm_pwrstat_telemetry.h new file mode 100644 index 0000000..455b6bf --- /dev/null +++ b/palacios/include/palacios/vmm_pwrstat_telemetry.h @@ -0,0 +1,52 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2013, The V3VEE Project + * All rights reserved. + * + * Author: Kyle C. Hale + * Chang S. Bae + * Peter Dinda + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#ifndef __VMM_PWRSTAT_TELEMETRY_H__ +#define __VMM_PWRSTAT_TELEMETRY_H__ + +#ifdef __V3VEE__ + +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + +#include +#include + +struct guest_info; + +#define PWRSTAT_NUM_COUNTERS 4 + +struct v3_core_pwrstat_telemetry { + enum {PWR_AWAIT_FIRST_ENTRY=0, PWR_AWAIT_ENTRY, PWR_AWAIT_EXIT} state; + uint8_t active_counters[PWRSTAT_NUM_COUNTERS]; + uint64_t guest_counts[PWRSTAT_NUM_COUNTERS]; + uint64_t host_counts[PWRSTAT_NUM_COUNTERS]; + uint64_t last_snapshot[PWRSTAT_NUM_COUNTERS]; +}; + + +void v3_pwrstat_telemetry_start(struct guest_info *info); +void v3_pwrstat_telemetry_enter(struct guest_info *info); +void v3_pwrstat_telemetry_exit(struct guest_info *info); +void v3_pwrstat_telemetry_end(struct guest_info *info); + + +#endif +#endif +#endif diff --git a/palacios/src/interfaces/Kconfig b/palacios/src/interfaces/Kconfig index 7535376..a72ce87 100644 --- a/palacios/src/interfaces/Kconfig +++ b/palacios/src/interfaces/Kconfig @@ -81,4 +81,11 @@ config HOST_PMU help Select this if you would like to access performance counters (the PMU) within Palacios + +config HOST_PWRSTAT + bool "Host power statistics monitoring support" + default n + help + Select this if you would like to access energy/power + measurements within Palacios endmenu diff --git a/palacios/src/interfaces/Makefile b/palacios/src/interfaces/Makefile index d0c7aa2..2e8b3e7 100644 --- a/palacios/src/interfaces/Makefile +++ b/palacios/src/interfaces/Makefile @@ -9,6 +9,7 @@ obj-$(V3_CONFIG_HOST_DEVICE) += vmm_host_dev.o obj-$(V3_CONFIG_HOST_HYPERCALL) += vmm_host_hypercall.o obj-$(V3_CONFIG_HOST_PCI) += host_pci.o obj-$(V3_CONFIG_HOST_PMU) += vmm_pmu.o +obj-$(V3_CONFIG_HOST_PWRSTAT) += vmm_pwrstat.o obj-y += null.o diff --git a/palacios/src/interfaces/vmm_pwrstat.c b/palacios/src/interfaces/vmm_pwrstat.c new file mode 100644 index 0000000..fff29bb --- /dev/null +++ b/palacios/src/interfaces/vmm_pwrstat.c @@ -0,0 +1,67 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2013, The V3VEE Project + * All rights reserved. + * + * Author: Kyle C. Hale + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#include +#include +#include +#include +#include + +#include + +struct v3_pwrstat_iface * palacios_pwrstat = 0; + + +void v3_pwrstat_init (void) +{ + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat != NULL); + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat->init != NULL); + palacios_pwrstat->init(); +} + + +void v3_pwrstat_deinit (void) +{ + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat != NULL); + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat->deinit != NULL); + palacios_pwrstat->deinit(); +} + + +uint64_t v3_pwrstat_get_value (v3_pwrstat_ctr_t ctr) +{ + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat != NULL); + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat->get_value != NULL); + return palacios_pwrstat->get_value(ctr); +} + + +int v3_pwrstat_ctr_valid(v3_pwrstat_ctr_t ctr) +{ + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat != NULL); + V3_ASSERT(VM_NONE, VCORE_NONE, palacios_pwrstat->ctr_valid != NULL); + return palacios_pwrstat->ctr_valid(ctr); +} + + +void V3_Init_Pwrstat (struct v3_pwrstat_iface * pwrstat_iface) +{ + palacios_pwrstat = pwrstat_iface; +} + + diff --git a/palacios/src/palacios/Makefile b/palacios/src/palacios/Makefile index e274ee1..cd53d35 100644 --- a/palacios/src/palacios/Makefile +++ b/palacios/src/palacios/Makefile @@ -81,6 +81,7 @@ obj-$(V3_CONFIG_CHECKPOINT) += vmm_checkpoint.o obj-$(V3_CONFIG_TELEMETRY) += vmm_telemetry.o obj-$(V3_CONFIG_PMU_TELEMETRY) += vmm_pmu_telemetry.o +obj-$(V3_CONFIG_PWRSTAT_TELEMETRY) += vmm_pwrstat_telemetry.o obj-$(V3_CONFIG_SYMBIOTIC) += vmm_symbiotic.o vmm_symspy.o obj-$(V3_CONFIG_SYMCALL) += vmm_symcall.o diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index b54ab19..847bb1e 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -694,9 +694,15 @@ int v3_svm_enter(struct guest_info * info) { uint64_t entry_tsc = 0; uint64_t exit_tsc = 0; +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_enter(info); +#endif + #ifdef V3_CONFIG_PMU_TELEMETRY v3_pmu_telemetry_enter(info); #endif + + rdtscll(entry_tsc); v3_svm_launch((vmcb_t *)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcbs[V3_Get_CPU()]); @@ -706,6 +712,11 @@ int v3_svm_enter(struct guest_info * info) { #ifdef V3_CONFIG_PMU_TELEMETRY v3_pmu_telemetry_exit(info); #endif + +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_exit(info); +#endif + guest_cycles = exit_tsc - entry_tsc; } @@ -839,6 +850,10 @@ int v3_start_svm_guest(struct guest_info * info) { v3_pmu_telemetry_start(info); #endif +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_start(info); +#endif + while (1) { if (info->vm_info->run_state == VM_STOPPED) { @@ -905,6 +920,10 @@ int v3_start_svm_guest(struct guest_info * info) { #ifdef V3_CONFIG_PMU_TELEMETRY v3_pmu_telemetry_end(info); #endif + +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_end(info); +#endif // Need to take down the other cores on error... return 0; diff --git a/palacios/src/palacios/vmm_pwrstat_telemetry.c b/palacios/src/palacios/vmm_pwrstat_telemetry.c new file mode 100644 index 0000000..98818c6 --- /dev/null +++ b/palacios/src/palacios/vmm_pwrstat_telemetry.c @@ -0,0 +1,196 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2013, The V3VEE Project + * All rights reserved. + * + * Author: Kyle C. Hale + * Chang S. Bae + * Peter Dinda + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ +#include +#include +#include +#include +#include + +/* + We will try to track: + + V3_PWRSTAT_PKG_ENERGY, + V3_PWRSTAT_CORE_ENERGY, + V3_PWRSTAT_EXT_ENERGY, + V3_PWRSTAT_DRAM_ENERGY, +*/ + +#define HAVE(WHAT) (info->pwrstat_telem.active_counters[WHAT]) + +#define GUEST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pwrstat_telem.guest_counts[WHAT]); } } while (0) +#define HOST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pwrstat_telem.host_counts[WHAT]); } } while (0) + +#define START(WHAT) \ +do { \ + if(v3_pwrstat_ctr_valid(WHAT)) { \ + info->pwrstat_telem.active_counters[WHAT]=1; \ + } else { \ + info->pwrstat_telem.active_counters[WHAT]=0;\ + } \ + } while (0) + +#define STOP(WHAT) info->pwrstat_telem.active_counters[WHAT]=0; + +static int print_pwrstat_data(struct guest_info *info, char * hdr) +{ + GUEST(V3_PWRSTAT_PKG_ENERGY); + GUEST(V3_PWRSTAT_CORE_ENERGY); + GUEST(V3_PWRSTAT_EXT_ENERGY); + GUEST(V3_PWRSTAT_DRAM_ENERGY); + + HOST(V3_PWRSTAT_PKG_ENERGY); + HOST(V3_PWRSTAT_CORE_ENERGY); + HOST(V3_PWRSTAT_EXT_ENERGY); + HOST(V3_PWRSTAT_DRAM_ENERGY); + + return 0; +} + + +static void telemetry_pwrstat (struct v3_vm_info * vm, void * private_data, char * hdr) +{ + int i; + struct guest_info *core = NULL; + + /* + * work through each pcore (vcore for now per excluding oversubscription) and gathering info + */ + for(i=0; inum_cores; i++) { + core = &(vm->cores[i]); + if((core->core_run_state != CORE_RUNNING)) continue; + print_pwrstat_data(core, hdr); + } +} + + +void v3_pwrstat_telemetry_start (struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + memset(&(info->pwrstat_telem),0,sizeof(struct v3_core_pwrstat_telemetry)); + + v3_pwrstat_init(); + + START(V3_PWRSTAT_PKG_ENERGY); + START(V3_PWRSTAT_CORE_ENERGY); + START(V3_PWRSTAT_EXT_ENERGY); + START(V3_PWRSTAT_DRAM_ENERGY); + + info->pwrstat_telem.state=PWR_AWAIT_FIRST_ENTRY; + + if (info->vcpu_id==0) { + v3_add_telemetry_cb(info->vm_info, telemetry_pwrstat, NULL); + } +} + + +static void inline snapshot(uint64_t vals[]) { + vals[V3_PWRSTAT_PKG_ENERGY] = v3_pwrstat_get_value(V3_PWRSTAT_PKG_ENERGY); + vals[V3_PWRSTAT_CORE_ENERGY] = v3_pwrstat_get_value(V3_PWRSTAT_CORE_ENERGY); + vals[V3_PWRSTAT_EXT_ENERGY] = v3_pwrstat_get_value(V3_PWRSTAT_EXT_ENERGY); + vals[V3_PWRSTAT_DRAM_ENERGY] = v3_pwrstat_get_value(V3_PWRSTAT_DRAM_ENERGY); +} + + +void v3_pwrstat_telemetry_enter(struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + switch (info->pwrstat_telem.state) { + case PWR_AWAIT_FIRST_ENTRY: + snapshot(info->pwrstat_telem.last_snapshot); + info->pwrstat_telem.state=PWR_AWAIT_EXIT; + break; + + case PWR_AWAIT_ENTRY: { + // AWAIT_ENTRY - the snapshot in the struct is from the last exit + uint64_t snap[PWRSTAT_NUM_COUNTERS]; + int i; + + snapshot(snap); + + for (i=0;ipwrstat_telem.host_counts[i] += snap[i] - info->pwrstat_telem.last_snapshot[i]; + } + + for (i=0;ipwrstat_telem.last_snapshot[i] = snap[i]; + } + + info->pwrstat_telem.state = PWR_AWAIT_EXIT; + } + break; + + default: + PrintError(info->vm_info, info, "Impossible state on pwrstat telemetry entry\n"); + break; + } + +} + + +void v3_pwrstat_telemetry_exit (struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + switch (info->pwrstat_telem.state) { + case PWR_AWAIT_EXIT: { + // AWAIT_EXIT - the snapshot in the struct is from the last entryx + uint64_t snap[PWRSTAT_NUM_COUNTERS]; + int i; + + snapshot(snap); + + for (i=0;ipwrstat_telem.guest_counts[i] += snap[i] - info->pwrstat_telem.last_snapshot[i]; + } + + for (i=0;ipwrstat_telem.last_snapshot[i] = snap[i]; + } + + info->pwrstat_telem.state = PWR_AWAIT_ENTRY; + } + break; + default: + PrintError(info->vm_info, info, "Impossible state on pwrstat telemetry exit\n"); + break; + } + +} + + +void v3_pwrstat_telemetry_end (struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + v3_pwrstat_deinit(); + + info->pwrstat_telem.state=PWR_AWAIT_FIRST_ENTRY; + +} diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index 6fd1f8d..2741051 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -1027,6 +1027,10 @@ int v3_vmx_enter(struct guest_info * info) { uint64_t entry_tsc = 0; uint64_t exit_tsc = 0; +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_enter(info); +#endif + #ifdef V3_CONFIG_PMU_TELEMETRY v3_pmu_telemetry_enter(info); #endif @@ -1049,6 +1053,10 @@ int v3_vmx_enter(struct guest_info * info) { #ifdef V3_CONFIG_PMU_TELEMETRY v3_pmu_telemetry_exit(info); #endif + +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_exit(info); +#endif } // PrintDebug(info->vm_info, info, "VMX Exit: ret=%d\n", ret); @@ -1202,6 +1210,10 @@ int v3_start_vmx_guest(struct guest_info * info) { v3_pmu_telemetry_start(info); #endif +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_start(info); +#endif + while (1) { if (info->vm_info->run_state == VM_STOPPED) { @@ -1262,6 +1274,10 @@ int v3_start_vmx_guest(struct guest_info * info) { v3_pmu_telemetry_end(info); #endif +#ifdef V3_CONFIG_PWRSTAT_TELEMETRY + v3_pwrstat_telemetry_end(info); +#endif + return 0; }