From: Jack Lange Date: Fri, 21 Aug 2009 19:45:42 +0000 (-0500) Subject: ported the profiler over to the telemetry interface X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=f3eb8bbb7c58c98b03797f2188e6c1d2a7610c15 ported the profiler over to the telemetry interface --- diff --git a/Kconfig b/Kconfig index 8050708..d682920 100644 --- a/Kconfig +++ b/Kconfig @@ -28,14 +28,33 @@ config SOCKET -config PROFILE_VMM - bool "Enable VMM profiling support" +config TELEMETRY + bool "Enable VMM telemetry support" default y help - Enable the profiling framework in Palacios + Enable the telemetry framework in Palacios ----- This is a framwork that allows components of palacios to record - information that is periodically reported to the log file + information that is periodically reported to the log output. + Telemetry is automatically collected for vmexits. Subsystem telemetry + is configured separately + + +config SHADOW_PAGING_TELEMETRY + bool "Enable Shadow Paging Telemetry" + default y + depends on TELEMETRY + help + Enable telemetry information for shadow paging + +config SYMBIOTIC_SWAP_TELEMETRY + bool "Enable Symbiotic Swap Telemetry" + default n + depends on TELEMETRY && SYMBIOTIC_SWAP + help + Enable the telemetry information for the symbiotic swap subsystem + + config INSTRUMENT_VMM bool "Enable VMM instrumentation" diff --git a/palacios/include/palacios/vm_guest.h b/palacios/include/palacios/vm_guest.h index 831d146..abf1cd6 100644 --- a/palacios/include/palacios/vm_guest.h +++ b/palacios/include/palacios/vm_guest.h @@ -35,8 +35,8 @@ #include -#ifdef CONFIG_PROFILE_VMM -#include +#ifdef CONFIG_TELEMETRY +#include #endif #ifdef CONFIG_SYMBIOTIC_SWAP @@ -118,8 +118,8 @@ struct v3_segments { struct shadow_page_state; struct v3_intr_state; -#ifdef CONFIG_PROFILE_VMM -struct v3_profiler; +#ifdef CONFIG_TELEMETRY +struct v3_telemetry; #endif #ifdef CONFIG_SYMBIOTIC_SWAP @@ -179,9 +179,9 @@ struct guest_info { uint64_t yield_start_cycle; -#ifdef CONFIG_PROFILE_VMM - uint_t enable_profiler; - struct v3_profiler profiler; +#ifdef CONFIG_TELEMETRY + uint_t enable_telemetry; + struct v3_telemetry_state telemetry; #endif diff --git a/palacios/include/palacios/vmm.h b/palacios/include/palacios/vmm.h index 878966f..2d32cbc 100644 --- a/palacios/include/palacios/vmm.h +++ b/palacios/include/palacios/vmm.h @@ -258,7 +258,7 @@ struct v3_vm_config { // so we can specify maximum physical address size // (We're screwed if we want to do 32 bit host/64 bit guest) - int enable_profiling; + int enable_telemetry; int enable_nested_paging; int enable_pci; diff --git a/palacios/include/palacios/vmm_shadow_paging.h b/palacios/include/palacios/vmm_shadow_paging.h index 0078e0b..cd0cb7a 100644 --- a/palacios/include/palacios/vmm_shadow_paging.h +++ b/palacios/include/palacios/vmm_shadow_paging.h @@ -40,6 +40,11 @@ struct shadow_page_state { // list of allocated shadow pages struct list_head page_list; + +#ifdef CONFIG_SHADOW_PAGING_TELEMETRY + uint_t guest_faults; +#endif + }; diff --git a/palacios/include/palacios/vmm_profiler.h b/palacios/include/palacios/vmm_telemetry.h similarity index 53% rename from palacios/include/palacios/vmm_profiler.h rename to palacios/include/palacios/vmm_telemetry.h index 9787db0..80ac874 100644 --- a/palacios/include/palacios/vmm_profiler.h +++ b/palacios/include/palacios/vmm_telemetry.h @@ -17,35 +17,45 @@ * redistribute, and modify it as specified in the file "V3VEE_LICENSE". */ -#ifndef __VMM_PROFILER_H__ -#define __VMM_PROFILER_H__ +#ifndef __VMM_TELEMETRY_H__ +#define __VMM_TELEMETRY_H__ #ifdef __V3VEE__ -#ifdef CONFIG_PROFILE_VMM +#ifdef CONFIG_TELEMETRY #include +#include struct guest_info; -struct v3_profiler { - uint_t total_exits; +struct v3_telemetry_state { - ullong_t start_time; - ullong_t end_time; + uint64_t vmm_start_tsc; + uint64_t prev_tsc; - uint_t guest_pf_cnt; + uint_t exit_cnt; + struct rb_root exit_root; - struct rb_root root; + uint32_t invoke_cnt; + uint64_t granularity; + + struct list_head cb_list; }; -void v3_init_profiler(struct guest_info * info); +void v3_init_telemetry(struct guest_info * info); + +void v3_telemetry_start_exit(struct guest_info * info); +void v3_telemetry_end_exit(struct guest_info * info, uint_t exit_code); + +void v3_print_telemetry(struct guest_info * info); -void v3_profile_exit(struct guest_info * info, uint_t exit_code); -void v3_print_profile(struct guest_info * info); +void v3_add_telemetry_cb(struct guest_info * info, + void (*telemetry_fn)(struct guest_info * info, void * private_data), + void * private_data); #endif diff --git a/palacios/src/palacios/Makefile b/palacios/src/palacios/Makefile index 88f917a..87ee4d8 100644 --- a/palacios/src/palacios/Makefile +++ b/palacios/src/palacios/Makefile @@ -51,7 +51,7 @@ obj-$(CONFIG_VMX) += vmx.o \ obj-$(CONFIG_INSTRUMENT_VMM) += vmm_instrument.o -obj-$(CONFIG_PROFILE_VMM) += vmm_profiler.o +obj-$(CONFIG_TELEMETRY) += vmm_telemetry.o obj-$(CONFIG_SOCKET) += vmm_socket.o obj-$(CONFIG_SYMBIOTIC_SWAP) += vmm_sym_swap.o \ No newline at end of file diff --git a/palacios/src/palacios/svm.c b/palacios/src/palacios/svm.c index 96ab2cf..37f509c 100644 --- a/palacios/src/palacios/svm.c +++ b/palacios/src/palacios/svm.c @@ -36,8 +36,6 @@ #include -#include - #include #include @@ -320,12 +318,6 @@ static int start_svm_guest(struct guest_info *info) { if ((num_exits % 5000) == 0) { PrintDebug("SVM Exit number %d\n", num_exits); - -#ifdef CONFIG_PROFILE_VMM - if (info->enable_profiler) { - v3_print_profile(info); - } -#endif } if (v3_handle_svm_exit(info) != 0) { diff --git a/palacios/src/palacios/svm_handler.c b/palacios/src/palacios/svm_handler.c index a70b0dd..30d581e 100644 --- a/palacios/src/palacios/svm_handler.c +++ b/palacios/src/palacios/svm_handler.c @@ -30,10 +30,12 @@ #include #include #include -#include #include #include +#ifdef CONFIG_TELEMETRY +#include +#endif int v3_handle_svm_exit(struct guest_info * info) { @@ -97,9 +99,9 @@ int v3_handle_svm_exit(struct guest_info * info) { } -#ifdef CONFIG_PROFILE_VMM - if (info->enable_profiler) { - rdtscll(info->profiler.start_time); +#ifdef CONFIG_TELEMETRY + if (info->enable_telemetry) { + v3_telemetry_start_exit(info); } #endif @@ -320,10 +322,9 @@ int v3_handle_svm_exit(struct guest_info * info) { } // END OF SWITCH (EXIT_CODE) -#ifdef CONFIG_PROFILE_VMM - if (info->enable_profiler) { - rdtscll(info->profiler.end_time); - v3_profile_exit(info, exit_code); +#ifdef CONFIG_TELEMETRY + if (info->enable_telemetry) { + v3_telemetry_end_exit(info, exit_code); } #endif diff --git a/palacios/src/palacios/vmm.c b/palacios/src/palacios/vmm.c index 4beccea..fea7009 100644 --- a/palacios/src/palacios/vmm.c +++ b/palacios/src/palacios/vmm.c @@ -141,3 +141,5 @@ void v3_yield(struct guest_info * info) { V3_Yield(); rdtscll(info->yield_start_cycle); } + + diff --git a/palacios/src/palacios/vmm_config.c b/palacios/src/palacios/vmm_config.c index b6e5587..af45a0a 100644 --- a/palacios/src/palacios/vmm_config.c +++ b/palacios/src/palacios/vmm_config.c @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include #include @@ -75,8 +75,19 @@ int v3_pre_config_guest(struct guest_info * info, struct v3_vm_config * config_p // Amount of ram the Guest will have, rounded to a 4K page boundary info->mem_size = config_ptr->mem_size & ~(addr_t)0xfff; + /* + * Initialize the subsystem data strutures + */ +#ifdef CONFIG_TELEMETRY + // This should go first, because other subsystems will depend on the guest_info flag + if (config_ptr->enable_telemetry) { + info->enable_telemetry = 1; + v3_init_telemetry(info); + } else { + info->enable_telemetry = 0; + } +#endif - // Initialize the subsystem data strutures v3_init_time(info); v3_init_io_map(info); v3_init_msr_map(info); @@ -109,14 +120,7 @@ int v3_pre_config_guest(struct guest_info * info, struct v3_vm_config * config_p info->shdw_pg_mode = SHADOW_PAGING; } -#ifdef CONFIG_PROFILE_VMM - if (config_ptr->enable_profiling) { - info->enable_profiler = 1; - v3_init_profiler(info); - } else { - info->enable_profiler = 0; - } -#endif + if (config_ptr->schedule_freq == 0) { // set the schedule frequency to 100 HZ diff --git a/palacios/src/palacios/vmm_profiler.c b/palacios/src/palacios/vmm_profiler.c deleted file mode 100644 index 567f652..0000000 --- a/palacios/src/palacios/vmm_profiler.c +++ /dev/null @@ -1,153 +0,0 @@ -/* - * This file is part of the Palacios Virtual Machine Monitor developed - * by the V3VEE Project with funding from the United States National - * Science Foundation and the Department of Energy. - * - * The V3VEE Project is a joint project between Northwestern University - * and the University of New Mexico. You can find out more at - * http://www.v3vee.org - * - * Copyright (c) 2008, Jack Lange - * Copyright (c) 2008, The V3VEE Project - * All rights reserved. - * - * Author: Jack Lange - * - * This is free software. You are permitted to use, - * redistribute, and modify it as specified in the file "V3VEE_LICENSE". - */ - -#include -#include -#include -#include - - -struct exit_event { - uint_t exit_code; - uint_t exit_count; - uint_t handler_time; - - struct rb_node tree_node; -}; - - -void v3_init_profiler(struct guest_info * info) { - info->profiler.total_exits = 0; - - info->profiler.start_time = 0; - info->profiler.end_time = 0; - info->profiler.guest_pf_cnt = 0; - - info->profiler.root.rb_node = NULL; -} - - - -static inline struct exit_event * __insert_event(struct guest_info * info, - struct exit_event * evt) { - struct rb_node ** p = &(info->profiler.root.rb_node); - struct rb_node * parent = NULL; - struct exit_event * tmp_evt = NULL; - - while (*p) { - parent = *p; - tmp_evt = rb_entry(parent, struct exit_event, tree_node); - - if (evt->exit_code < tmp_evt->exit_code) { - p = &(*p)->rb_left; - } else if (evt->exit_code > tmp_evt->exit_code) { - p = &(*p)->rb_right; - } else { - return tmp_evt; - } - } - rb_link_node(&(evt->tree_node), parent, p); - - return NULL; -} - -static inline struct exit_event * insert_event(struct guest_info * info, - struct exit_event * evt) { - struct exit_event * ret; - - if ((ret = __insert_event(info, evt))) { - return ret; - } - - v3_rb_insert_color(&(evt->tree_node), &(info->profiler.root)); - - return NULL; -} - - -static struct exit_event * get_exit(struct guest_info * info, uint_t exit_code) { - struct rb_node * n = info->profiler.root.rb_node; - struct exit_event * evt = NULL; - - while (n) { - evt = rb_entry(n, struct exit_event, tree_node); - - if (exit_code < evt->exit_code) { - n = n->rb_left; - } else if (exit_code > evt->exit_code) { - n = n->rb_right; - } else { - return evt; - } - } - - return NULL; -} - - -static inline struct exit_event * create_exit(uint_t exit_code) { - struct exit_event * evt = V3_Malloc(sizeof(struct exit_event)); - - evt->exit_code = exit_code; - evt->exit_count = 0; - evt->handler_time = 0; - - return evt; -} - -void v3_profile_exit(struct guest_info * info, uint_t exit_code) { - uint_t time = (info->profiler.end_time - info->profiler.start_time); - struct exit_event * evt = get_exit(info, exit_code); - - if (evt == NULL) { - evt = create_exit(exit_code); - insert_event(info, evt); - } - - - - - evt->handler_time = (evt->handler_time * 127ull + time) / 128; - - - evt->exit_count++; - - info->profiler.total_exits++; -} - - -void v3_print_profile(struct guest_info * info) { - struct exit_event * evt = NULL; - struct rb_node * node = v3_rb_first(&(info->profiler.root)); - - PrintDebug("GUEST_PF: %u\n", info->profiler.guest_pf_cnt); - - do { - evt = rb_entry(node, struct exit_event, tree_node); - const char * code_str = vmexit_code_to_str(evt->exit_code); - - PrintDebug("%s:%sCnt=%u,%sTime=%u\n", - code_str, - (strlen(code_str) > 14) ? "\t" : "\t\t", - evt->exit_count, - (evt->exit_count >= 100) ? "\t" : "\t\t", - evt->handler_time); - - } while ((node = v3_rb_next(node))); -} diff --git a/palacios/src/palacios/vmm_shadow_paging.c b/palacios/src/palacios/vmm_shadow_paging.c index eeb493b..eb2fe6b 100644 --- a/palacios/src/palacios/vmm_shadow_paging.c +++ b/palacios/src/palacios/vmm_shadow_paging.c @@ -30,6 +30,11 @@ #include + +#ifdef CONFIG_SHADOW_PAGING_TELEMETRY +#include +#endif + #ifdef CONFIG_SYMBIOTIC_SWAP #include #endif @@ -65,6 +70,14 @@ static int is_guest_pf(pt_access_status_t guest_access, pt_access_status_t shado +#ifdef CONFIG_SHADOW_PAGING_TELEMETRY +static void telemetry_cb(struct guest_info * info, void * private_data) { + V3_Print("Guest Page faults: %d\n", info->shdw_pg_state.guest_faults); +} +#endif + + + int v3_init_shadow_page_state(struct guest_info * info) { struct shadow_page_state * state = &(info->shdw_pg_state); @@ -73,6 +86,12 @@ int v3_init_shadow_page_state(struct guest_info * info) { state->guest_efer.value = 0x0LL; INIT_LIST_HEAD(&(state->page_list)); + +#ifdef CONFIG_SHADOW_PAGING_TELEMETRY + if (info->enable_telemetry) { + v3_add_telemetry_cb(info, telemetry_cb, NULL); + } +#endif return 0; } @@ -241,15 +260,12 @@ static struct shadow_page_data * create_new_shadow_pt(struct guest_info * info) static int inject_guest_pf(struct guest_info * info, addr_t fault_addr, pf_error_t error_code) { + info->ctrl_regs.cr2 = fault_addr; -#ifdef CONFIG_PROFILE_VMM - if (info->enable_profiler) { - info->profiler.guest_pf_cnt++; - } +#ifdef CONFIG_SHADOW_PAGING_TELEMETRY + info->shdw_pg_state.guest_faults++; #endif - info->ctrl_regs.cr2 = fault_addr; - return v3_raise_exception_with_error(info, PF_EXCEPTION, *(uint_t *)&error_code); } diff --git a/palacios/src/palacios/vmm_telemetry.c b/palacios/src/palacios/vmm_telemetry.c new file mode 100644 index 0000000..906f2ef --- /dev/null +++ b/palacios/src/palacios/vmm_telemetry.c @@ -0,0 +1,222 @@ +/* + * This file is part of the Palacios Virtual Machine Monitor developed + * by the V3VEE Project with funding from the United States National + * Science Foundation and the Department of Energy. + * + * The V3VEE Project is a joint project between Northwestern University + * and the University of New Mexico. You can find out more at + * http://www.v3vee.org + * + * Copyright (c) 2008, Jack Lange + * Copyright (c) 2008, The V3VEE Project + * All rights reserved. + * + * Author: Jack Lange + * + * This is free software. You are permitted to use, + * redistribute, and modify it as specified in the file "V3VEE_LICENSE". + */ + +#include +#include +#include +#include + + +#ifdef CONFIG_TELEMETRY_GRANULARITY +#define DEFAULT_GRANULARITY CONFIG_TELEMETRY_GRANULARITY +#else +#define DEFAULT_GRANULARITY 5000 +#endif + + + +struct telemetry_cb { + + void (*telemetry_fn)(struct guest_info * info, void * private_data); + + void * private_data; + struct list_head cb_node; +}; + + +struct exit_event { + uint_t exit_code; + uint_t cnt; + uint64_t handler_time; + + struct rb_node tree_node; +}; + + +void v3_init_telemetry(struct guest_info * info) { + struct v3_telemetry_state * telemetry = &(info->telemetry); + + telemetry->exit_cnt = 0; + telemetry->vmm_start_tsc = 0; + telemetry->prev_tsc = 0; + telemetry->invoke_cnt = 0; + telemetry->granularity = DEFAULT_GRANULARITY; + + + telemetry->exit_root.rb_node = NULL; + INIT_LIST_HEAD(&(telemetry->cb_list)); +} + + + +static inline struct exit_event * __insert_event(struct guest_info * info, + struct exit_event * evt) { + struct rb_node ** p = &(info->telemetry.exit_root.rb_node); + struct rb_node * parent = NULL; + struct exit_event * tmp_evt = NULL; + + while (*p) { + parent = *p; + tmp_evt = rb_entry(parent, struct exit_event, tree_node); + + if (evt->exit_code < tmp_evt->exit_code) { + p = &(*p)->rb_left; + } else if (evt->exit_code > tmp_evt->exit_code) { + p = &(*p)->rb_right; + } else { + return tmp_evt; + } + } + rb_link_node(&(evt->tree_node), parent, p); + + return NULL; +} + +static inline struct exit_event * insert_event(struct guest_info * info, + struct exit_event * evt) { + struct exit_event * ret; + + if ((ret = __insert_event(info, evt))) { + return ret; + } + + v3_rb_insert_color(&(evt->tree_node), &(info->telemetry.exit_root)); + + return NULL; +} + + +static struct exit_event * get_exit(struct guest_info * info, uint_t exit_code) { + struct rb_node * n = info->telemetry.exit_root.rb_node; + struct exit_event * evt = NULL; + + while (n) { + evt = rb_entry(n, struct exit_event, tree_node); + + if (exit_code < evt->exit_code) { + n = n->rb_left; + } else if (exit_code > evt->exit_code) { + n = n->rb_right; + } else { + return evt; + } + } + + return NULL; +} + + +static inline struct exit_event * create_exit(uint_t exit_code) { + struct exit_event * evt = V3_Malloc(sizeof(struct exit_event)); + + evt->exit_code = exit_code; + evt->cnt = 0; + evt->handler_time = 0; + + return evt; +} + +void v3_telemetry_start_exit(struct guest_info * info) { + rdtscll(info->telemetry.vmm_start_tsc); +} + + +void v3_telemetry_end_exit(struct guest_info * info, uint_t exit_code) { + struct v3_telemetry_state * telemetry = &(info->telemetry); + struct exit_event * evt = NULL; + uint64_t end_tsc = 0; + + rdtscll(end_tsc); + + evt = get_exit(info, exit_code); + + if (evt == NULL) { + evt = create_exit(exit_code); + insert_event(info, evt); + } + + evt->handler_time += end_tsc - telemetry->vmm_start_tsc; + + evt->cnt++; + telemetry->exit_cnt++; + + + + // check if the exit count has expired + if ((telemetry->exit_cnt % telemetry->granularity) == 0) { + v3_print_telemetry(info); + } +} + + + + +void v3_add_telemetry_cb(struct guest_info * info, + void (*telemetry_fn)(struct guest_info * info, void * private_data), + void * private_data) { + struct v3_telemetry_state * telemetry = &(info->telemetry); + struct telemetry_cb * cb = (struct telemetry_cb *)V3_Malloc(sizeof(struct telemetry_cb)); + + cb->private_data = private_data; + cb->telemetry_fn = telemetry_fn; + + list_add(&(cb->cb_node), &(telemetry->cb_list)); +} + + +void v3_print_telemetry(struct guest_info * info) { + struct v3_telemetry_state * telemetry = &(info->telemetry); + uint64_t invoke_tsc = 0; + + rdtscll(invoke_tsc); + + V3_Print("Telemetry (%d)\n", telemetry->invoke_cnt++); + V3_Print("\ttelemetry window tsc cnt: %d\n", (uint32_t)(invoke_tsc - telemetry->prev_tsc)); + + // Exit Telemetry + { + struct exit_event * evt = NULL; + struct rb_node * node = v3_rb_first(&(info->telemetry.exit_root)); + + do { + evt = rb_entry(node, struct exit_event, tree_node); + const char * code_str = vmexit_code_to_str(evt->exit_code); + + V3_Print("%s:%sCnt=%u,%sAvg. Time=%u\n", + code_str, + (strlen(code_str) > 14) ? "\t" : "\t\t", + evt->cnt, + (evt->cnt >= 100) ? "\t" : "\t\t", + (uint32_t)(evt->handler_time / evt->cnt)); + + } while ((node = v3_rb_next(node))); + } + + + // Registered callbacks + { + struct telemetry_cb * cb = NULL; + + list_for_each_entry(cb, &(telemetry->cb_list), cb_node) { + cb->telemetry_fn(info, cb->private_data); + } + } + + telemetry->prev_tsc = invoke_tsc; +}