From: Peter Dinda Date: Wed, 20 Feb 2013 02:14:28 +0000 (-0600) Subject: PMU-based telemetry extension to monitor guest and host behavior X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?a=commitdiff_plain;h=d26b8881ecd0481b7fb5e24006fe2d84540ed3c1;p=palacios.releases.git PMU-based telemetry extension to monitor guest and host behavior --- diff --git a/Kconfig b/Kconfig index 5edd454..c44bc12 100644 --- a/Kconfig +++ b/Kconfig @@ -157,6 +157,14 @@ config SHADOW_PAGING_TELEMETRY help Enable telemetry information for shadow paging +config PMU_TELEMETRY + bool "Enable PMU telemetry" + default n + depends on TELEMETRY && HOST_PMU + help + Enable telemetry information for a range of PMU counters + This causes the currently configured PMU counts to be printed + config EXPERIMENTAL diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 66ac526..1e999ba 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -49,6 +49,9 @@ #include #endif +#ifdef V3_CONFIG_PMU_TELEMETRY +#include +#endif #ifdef V3_CONFIG_SYMBIOTIC #include @@ -119,6 +122,9 @@ struct guest_info { struct v3_core_telemetry core_telem; #endif +#ifdef V3_CONFIG_PMU_TELEMETRY + struct v3_core_pmu_telemetry pmu_telem; +#endif /* struct v3_core_dev_mgr core_dev_mgr; */ diff --git a/palacios/include/palacios/vmm_pmu_telemetry.h b/palacios/include/palacios/vmm_pmu_telemetry.h new file mode 100644 index 0000000..67518c8 --- /dev/null +++ b/palacios/include/palacios/vmm_pmu_telemetry.h @@ -0,0 +1,55 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2013, The V3VEE Project + * All rights reserved. + * + * Author: Chang S. Bae + * Peter Dinda + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#ifndef __VMM_PMU_TELEMETRY_H__ +#define __VMM_PMU_TELEMETRY_H__ + +#ifdef __V3VEE__ + +#ifdef V3_CONFIG_PMU_TELEMETRY + +#include +#include + +struct guest_info; + +#define PMU_NUM_COUNTERS 6 + +struct v3_core_pmu_telemetry { + enum {AWAIT_FIRST_ENTRY=0, AWAIT_ENTRY, AWAIT_EXIT} state; + uint8_t active_counters[PMU_NUM_COUNTERS]; + uint64_t guest_counts[PMU_NUM_COUNTERS]; + uint64_t host_counts[PMU_NUM_COUNTERS]; + uint64_t last_snapshot[PMU_NUM_COUNTERS]; + uint64_t guest_ucpi_estimate; // cycles per instruction * 1e6 + uint64_t guest_umpl_estimate; // cache misses per load * 1e6 + uint64_t host_ucpi_estimate; // cycles per instruction * 1e6 + uint64_t host_umpl_estimate; // cache misses per load * 1e6 +}; + + +void v3_pmu_telemetry_start(struct guest_info *info); +void v3_pmu_telemetry_enter(struct guest_info *info); +void v3_pmu_telemetry_exit(struct guest_info *info); +void v3_pmu_telemetry_end(struct guest_info *info); + + +#endif +#endif +#endif diff --git a/palacios/src/palacios/Makefile b/palacios/src/palacios/Makefile index c51a37e..777d8a8 100644 --- a/palacios/src/palacios/Makefile +++ b/palacios/src/palacios/Makefile @@ -79,6 +79,8 @@ obj-$(V3_CONFIG_CHECKPOINT) += vmm_checkpoint.o obj-$(V3_CONFIG_TELEMETRY) += vmm_telemetry.o +obj-$(V3_CONFIG_PMU_TELEMETRY) += vmm_pmu_telemetry.o + obj-$(V3_CONFIG_SYMBIOTIC) += vmm_symbiotic.o vmm_symspy.o obj-$(V3_CONFIG_SYMCALL) += vmm_symcall.o obj-$(V3_CONFIG_SYMMOD) += vmm_symmod.o diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index 448e5af..b54ab19 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -694,12 +694,18 @@ int v3_svm_enter(struct guest_info * info) { uint64_t entry_tsc = 0; uint64_t exit_tsc = 0; +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_enter(info); +#endif rdtscll(entry_tsc); v3_svm_launch((vmcb_t *)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcbs[V3_Get_CPU()]); rdtscll(exit_tsc); +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_exit(info); +#endif guest_cycles = exit_tsc - entry_tsc; } @@ -829,6 +835,10 @@ int v3_start_svm_guest(struct guest_info * info) { v3_start_time(info); +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_start(info); +#endif + while (1) { if (info->vm_info->run_state == VM_STOPPED) { @@ -892,6 +902,9 @@ int v3_start_svm_guest(struct guest_info * info) { } +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_end(info); +#endif // Need to take down the other cores on error... return 0; diff --git a/palacios/src/palacios/vmm_pmu_telemetry.c b/palacios/src/palacios/vmm_pmu_telemetry.c new file mode 100644 index 0000000..20bf3ec --- /dev/null +++ b/palacios/src/palacios/vmm_pmu_telemetry.c @@ -0,0 +1,280 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2013, The V3VEE Project + * All rights reserved. + * + * Author: Chang S. Bae + * Peter Dinda + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ +#include +#include +#include +#include + +/* + We will try to track: + + V3_PMON_RETIRED_INST_COUNT, + V3_PMON_CLOCK_COUNT, + V3_PMON_MEM_LOAD_COUNT, + V3_PMON_MEM_STORE_COUNT, + V3_PMON_CACHE_MISS_COUNT, + V3_PMON_TLB_MISS_COUNT + + and to derive: + + CPI + cache Misses per instruction +*/ + +#define HAVE(WHAT) (info->pmu_telem.active_counters[WHAT]) + +#define GUEST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pmu_telem.guest_counts[WHAT]); } } while (0) +#define HOST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pmu_telem.host_counts[WHAT]); } } while (0) + + +static int print_pmu_data(struct guest_info *info, char * hdr) +{ + GUEST(V3_PMON_RETIRED_INST_COUNT); + GUEST(V3_PMON_CLOCK_COUNT); + GUEST(V3_PMON_MEM_LOAD_COUNT); + GUEST(V3_PMON_MEM_STORE_COUNT); + GUEST(V3_PMON_CACHE_MISS_COUNT); + GUEST(V3_PMON_TLB_MISS_COUNT); + V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UCPI",info->pmu_telem.guest_ucpi_estimate); + V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UMPL", info->pmu_telem.guest_umpl_estimate); + + HOST(V3_PMON_RETIRED_INST_COUNT); + HOST(V3_PMON_CLOCK_COUNT); + HOST(V3_PMON_MEM_LOAD_COUNT); + HOST(V3_PMON_MEM_STORE_COUNT); + HOST(V3_PMON_CACHE_MISS_COUNT); + HOST(V3_PMON_TLB_MISS_COUNT); + V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UCPI",info->pmu_telem.host_ucpi_estimate); + V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UMPL", info->pmu_telem.host_umpl_estimate); + + return 0; +} + + +static void telemetry_pmu(struct v3_vm_info * vm, void * private_data, char * hdr) +{ + int i; + struct guest_info *core = NULL; + + /* + * work through each pcore (vcore for now per excluding oversubscription) and gathering info + */ + for(i=0; inum_cores; i++) { + core = &(vm->cores[i]); + if((core->core_run_state != CORE_RUNNING)) continue; + print_pmu_data(core, hdr); + } +} + + +#define START(WHAT) \ +do { \ + if(v3_pmu_start_tracking(WHAT) == -1) { \ + PrintError(info->vm_info, info, "Failed to start tracking of %s\n", #WHAT); \ + info->pmu_telem.active_counters[WHAT]=0; \ + } else { \ + info->pmu_telem.active_counters[WHAT]=1;\ + } \ + } while (0) + +#define STOP(WHAT) \ +do { \ + if(info->pmu_telem.active_counters[WHAT]) { \ + if (v3_pmu_stop_tracking(WHAT) == -1) { \ + PrintError(info->vm_info, info, "Failed to stop tracking of %s\n", #WHAT); \ + } \ + info->pmu_telem.active_counters[WHAT]=0; \ + } \ + } while (0) + + +void v3_pmu_telemetry_start(struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + memset(&(info->pmu_telem),0,sizeof(struct v3_core_pmu_telemetry)); + + v3_pmu_init(); + + START(V3_PMON_RETIRED_INST_COUNT); + START(V3_PMON_CLOCK_COUNT); + START(V3_PMON_MEM_LOAD_COUNT); + START(V3_PMON_MEM_STORE_COUNT); + START(V3_PMON_CACHE_MISS_COUNT); + START(V3_PMON_TLB_MISS_COUNT); + + + info->pmu_telem.state=AWAIT_FIRST_ENTRY; + + + if (info->vcpu_id==0) { + v3_add_telemetry_cb(info->vm_info, telemetry_pmu, NULL); + } + +} + +static void inline snapshot(uint64_t vals[]) { + vals[V3_PMON_RETIRED_INST_COUNT] = v3_pmu_get_value(V3_PMON_RETIRED_INST_COUNT); + vals[V3_PMON_CLOCK_COUNT] = v3_pmu_get_value(V3_PMON_CLOCK_COUNT); + vals[V3_PMON_MEM_LOAD_COUNT] = v3_pmu_get_value(V3_PMON_MEM_LOAD_COUNT); + vals[V3_PMON_MEM_STORE_COUNT] = v3_pmu_get_value(V3_PMON_MEM_STORE_COUNT); + vals[V3_PMON_CACHE_MISS_COUNT] = v3_pmu_get_value(V3_PMON_CACHE_MISS_COUNT); + vals[V3_PMON_TLB_MISS_COUNT] = v3_pmu_get_value(V3_PMON_TLB_MISS_COUNT); +} + + +#define ALPHA_DENOM 8 // we are counting in 8ths +#define ALPHA_NUM 1 // 1/8 to new value +#define OM_ALPHA_NUM 7 // 7/8 to estimate + +static inline void update_ucpi_estimate(uint64_t *estimate, uint64_t counts[], uint64_t last[]) +{ + // 1e6 times the number of cycles since last + uint64_t ucycles = 1000000 * (counts[V3_PMON_CLOCK_COUNT] - last[V3_PMON_CLOCK_COUNT]); + uint64_t insts = counts[V3_PMON_RETIRED_INST_COUNT] - last[V3_PMON_RETIRED_INST_COUNT]; + + if (insts==0) { + return; + } + + *estimate = ((ALPHA_NUM * (*estimate)) + (OM_ALPHA_NUM * ((ucycles/insts)))) / ALPHA_DENOM; + +} + +static inline void update_umpl_estimate(uint64_t *estimate, uint64_t counts[], uint64_t last[]) +{ + // 1e6 times the number of misses since the last time + uint64_t umisses = 1000000 * (counts[V3_PMON_CACHE_MISS_COUNT] - last[V3_PMON_CACHE_MISS_COUNT]); + uint64_t loads = counts[V3_PMON_MEM_LOAD_COUNT] - last[V3_PMON_MEM_LOAD_COUNT]; + + if (loads==0) { + return; + } + + *estimate = ((ALPHA_NUM * (*estimate)) + (OM_ALPHA_NUM * ((umisses/loads)))) / ALPHA_DENOM; + +} + +void v3_pmu_telemetry_enter(struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + switch (info->pmu_telem.state) { + case AWAIT_FIRST_ENTRY: + snapshot(info->pmu_telem.last_snapshot); + info->pmu_telem.state=AWAIT_EXIT; + break; + + case AWAIT_ENTRY: { + // AWAIT_ENTRY - the snapshot in the struct is from the last exit + uint64_t snap[PMU_NUM_COUNTERS]; + int i; + + snapshot(snap); + + for (i=0;ipmu_telem.host_counts[i] += snap[i] - info->pmu_telem.last_snapshot[i]; + } + + if (HAVE(V3_PMON_CLOCK_COUNT) && HAVE(V3_PMON_RETIRED_INST_COUNT)) { + update_ucpi_estimate(&(info->pmu_telem.host_ucpi_estimate), info->pmu_telem.host_counts, info->pmu_telem.last_snapshot); + } + if (HAVE(V3_PMON_CACHE_MISS_COUNT) && HAVE(V3_PMON_MEM_LOAD_COUNT)) { + update_umpl_estimate(&(info->pmu_telem.host_umpl_estimate), info->pmu_telem.host_counts, info->pmu_telem.last_snapshot); + } + + for (i=0;ipmu_telem.last_snapshot[i] = snap[i]; + } + + info->pmu_telem.state = AWAIT_EXIT; + } + break; + + default: + PrintError(info->vm_info, info, "Impossible state in on pmu telemetry entry\n"); + break; + } + +} + + + +void v3_pmu_telemetry_exit(struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + switch (info->pmu_telem.state) { + case AWAIT_EXIT: { + // AWAIT_EXIT - the snapshot in the struct is from the last entryx + uint64_t snap[PMU_NUM_COUNTERS]; + int i; + + snapshot(snap); + + for (i=0;ipmu_telem.guest_counts[i] += snap[i] - info->pmu_telem.last_snapshot[i]; + } + + if (HAVE(V3_PMON_CLOCK_COUNT) && HAVE(V3_PMON_RETIRED_INST_COUNT)) { + update_ucpi_estimate(&(info->pmu_telem.guest_ucpi_estimate), info->pmu_telem.guest_counts, info->pmu_telem.last_snapshot); + } + if (HAVE(V3_PMON_CACHE_MISS_COUNT) && HAVE(V3_PMON_MEM_LOAD_COUNT)) { + update_umpl_estimate(&(info->pmu_telem.guest_umpl_estimate), info->pmu_telem.guest_counts, info->pmu_telem.last_snapshot); + } + + for (i=0;ipmu_telem.last_snapshot[i] = snap[i]; + } + + info->pmu_telem.state = AWAIT_ENTRY; + } + break; + default: + PrintError(info->vm_info, info, "Impossible state in on pmu telemetry exit\n"); + break; + } + +} + +void v3_pmu_telemetry_end(struct guest_info *info) +{ + if (!info->vm_info->enable_telemetry) { + return; + } + + STOP(V3_PMON_RETIRED_INST_COUNT); + STOP(V3_PMON_CLOCK_COUNT); + STOP(V3_PMON_MEM_LOAD_COUNT); + STOP(V3_PMON_MEM_STORE_COUNT); + STOP(V3_PMON_CACHE_MISS_COUNT); + STOP(V3_PMON_TLB_MISS_COUNT); + + v3_pmu_deinit(); + + info->pmu_telem.state=AWAIT_FIRST_ENTRY; + + // Umm.... there is no v3_remove_telemtry_cb ? WTF? +} diff --git a/palacios/src/palacios/vmx.c b/palacios/src/palacios/vmx.c index c74607e..6fd1f8d 100644 --- a/palacios/src/palacios/vmx.c +++ b/palacios/src/palacios/vmx.c @@ -1027,6 +1027,10 @@ int v3_vmx_enter(struct guest_info * info) { uint64_t entry_tsc = 0; uint64_t exit_tsc = 0; +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_enter(info); +#endif + if (vmx_info->state == VMX_UNLAUNCHED) { vmx_info->state = VMX_LAUNCHED; rdtscll(entry_tsc); @@ -1041,6 +1045,10 @@ int v3_vmx_enter(struct guest_info * info) { } guest_cycles = exit_tsc - entry_tsc; + +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_exit(info); +#endif } // PrintDebug(info->vm_info, info, "VMX Exit: ret=%d\n", ret); @@ -1190,6 +1198,10 @@ int v3_start_vmx_guest(struct guest_info * info) { v3_start_time(info); +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_start(info); +#endif + while (1) { if (info->vm_info->run_state == VM_STOPPED) { @@ -1246,6 +1258,10 @@ int v3_start_vmx_guest(struct guest_info * info) { } +#ifdef V3_CONFIG_PMU_TELEMETRY + v3_pmu_telemetry_end(info); +#endif + return 0; }