2 * This file is part of the Palacios Virtual Machine Monitor developed
3 * by the V3VEE Project with funding from the United States National
4 * Science Foundation and the Department of Energy.
6 * The V3VEE Project is a joint project between Northwestern University
7 * and the University of New Mexico. You can find out more at
10 * Copyright (c) 2013, The V3VEE Project <http://www.v3vee.org>
11 * All rights reserved.
13 * Author: Chang S. Bae <chang.bae@eecs.northwestern.edu>
14 * Peter Dinda <pdinda@northwestern.edu>
16 * This is free software. You are permitted to use,
17 * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
19 #include <palacios/vm_guest.h>
20 #include <palacios/vmm_telemetry.h>
21 #include <palacios/vmm_pmu_telemetry.h>
22 #include <palacios/vmm_sprintf.h>
27 V3_PMON_RETIRED_INST_COUNT,
29 V3_PMON_MEM_LOAD_COUNT,
30 V3_PMON_MEM_STORE_COUNT,
31 V3_PMON_CACHE_MISS_COUNT,
32 V3_PMON_TLB_MISS_COUNT
37 cache Misses per instruction
40 #define HAVE(WHAT) (info->pmu_telem.active_counters[WHAT])
42 #define GUEST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pmu_telem.guest_counts[WHAT]); } } while (0)
43 #define HOST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pmu_telem.host_counts[WHAT]); } } while (0)
46 static int print_pmu_data(struct guest_info *info, char * hdr)
48 GUEST(V3_PMON_RETIRED_INST_COUNT);
49 GUEST(V3_PMON_CLOCK_COUNT);
50 GUEST(V3_PMON_MEM_LOAD_COUNT);
51 GUEST(V3_PMON_MEM_STORE_COUNT);
52 GUEST(V3_PMON_CACHE_MISS_COUNT);
53 GUEST(V3_PMON_TLB_MISS_COUNT);
54 V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UCPI",info->pmu_telem.guest_ucpi_estimate);
55 V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UMPL", info->pmu_telem.guest_umpl_estimate);
57 HOST(V3_PMON_RETIRED_INST_COUNT);
58 HOST(V3_PMON_CLOCK_COUNT);
59 HOST(V3_PMON_MEM_LOAD_COUNT);
60 HOST(V3_PMON_MEM_STORE_COUNT);
61 HOST(V3_PMON_CACHE_MISS_COUNT);
62 HOST(V3_PMON_TLB_MISS_COUNT);
63 V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UCPI",info->pmu_telem.host_ucpi_estimate);
64 V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UMPL", info->pmu_telem.host_umpl_estimate);
70 static void telemetry_pmu(struct v3_vm_info * vm, void * private_data, char * hdr)
73 struct guest_info *core = NULL;
76 * work through each pcore (vcore for now per excluding oversubscription) and gathering info
78 for(i=0; i<vm->num_cores; i++) {
79 core = &(vm->cores[i]);
80 if((core->core_run_state != CORE_RUNNING)) continue;
81 print_pmu_data(core, hdr);
88 if(v3_pmu_start_tracking(WHAT) == -1) { \
89 PrintError(info->vm_info, info, "Failed to start tracking of %s\n", #WHAT); \
90 info->pmu_telem.active_counters[WHAT]=0; \
92 info->pmu_telem.active_counters[WHAT]=1;\
98 if(info->pmu_telem.active_counters[WHAT]) { \
99 if (v3_pmu_stop_tracking(WHAT) == -1) { \
100 PrintError(info->vm_info, info, "Failed to stop tracking of %s\n", #WHAT); \
102 info->pmu_telem.active_counters[WHAT]=0; \
107 void v3_pmu_telemetry_start(struct guest_info *info)
109 if (!info->vm_info->enable_telemetry) {
113 memset(&(info->pmu_telem),0,sizeof(struct v3_core_pmu_telemetry));
117 START(V3_PMON_RETIRED_INST_COUNT);
118 START(V3_PMON_CLOCK_COUNT);
119 START(V3_PMON_MEM_LOAD_COUNT);
120 START(V3_PMON_MEM_STORE_COUNT);
121 START(V3_PMON_CACHE_MISS_COUNT);
122 START(V3_PMON_TLB_MISS_COUNT);
125 info->pmu_telem.state=AWAIT_FIRST_ENTRY;
128 if (info->vcpu_id==0) {
129 v3_add_telemetry_cb(info->vm_info, telemetry_pmu, NULL);
134 static void inline snapshot(uint64_t vals[]) {
135 vals[V3_PMON_RETIRED_INST_COUNT] = v3_pmu_get_value(V3_PMON_RETIRED_INST_COUNT);
136 vals[V3_PMON_CLOCK_COUNT] = v3_pmu_get_value(V3_PMON_CLOCK_COUNT);
137 vals[V3_PMON_MEM_LOAD_COUNT] = v3_pmu_get_value(V3_PMON_MEM_LOAD_COUNT);
138 vals[V3_PMON_MEM_STORE_COUNT] = v3_pmu_get_value(V3_PMON_MEM_STORE_COUNT);
139 vals[V3_PMON_CACHE_MISS_COUNT] = v3_pmu_get_value(V3_PMON_CACHE_MISS_COUNT);
140 vals[V3_PMON_TLB_MISS_COUNT] = v3_pmu_get_value(V3_PMON_TLB_MISS_COUNT);
144 #define ALPHA_DENOM 8 // we are counting in 8ths
145 #define ALPHA_NUM 1 // 1/8 to new value
146 #define OM_ALPHA_NUM 7 // 7/8 to estimate
148 static inline void update_ucpi_estimate(uint64_t *estimate, uint64_t counts[], uint64_t last[])
150 // 1e6 times the number of cycles since last
151 uint64_t ucycles = 1000000 * (counts[V3_PMON_CLOCK_COUNT] - last[V3_PMON_CLOCK_COUNT]);
152 uint64_t insts = counts[V3_PMON_RETIRED_INST_COUNT] - last[V3_PMON_RETIRED_INST_COUNT];
158 *estimate = ((ALPHA_NUM * (*estimate)) + (OM_ALPHA_NUM * ((ucycles/insts)))) / ALPHA_DENOM;
162 static inline void update_umpl_estimate(uint64_t *estimate, uint64_t counts[], uint64_t last[])
164 // 1e6 times the number of misses since the last time
165 uint64_t umisses = 1000000 * (counts[V3_PMON_CACHE_MISS_COUNT] - last[V3_PMON_CACHE_MISS_COUNT]);
166 uint64_t loads = counts[V3_PMON_MEM_LOAD_COUNT] - last[V3_PMON_MEM_LOAD_COUNT];
172 *estimate = ((ALPHA_NUM * (*estimate)) + (OM_ALPHA_NUM * ((umisses/loads)))) / ALPHA_DENOM;
176 void v3_pmu_telemetry_enter(struct guest_info *info)
178 if (!info->vm_info->enable_telemetry) {
182 switch (info->pmu_telem.state) {
183 case AWAIT_FIRST_ENTRY:
184 snapshot(info->pmu_telem.last_snapshot);
185 info->pmu_telem.state=AWAIT_EXIT;
189 // AWAIT_ENTRY - the snapshot in the struct is from the last exit
190 uint64_t snap[PMU_NUM_COUNTERS];
195 for (i=0;i<PMU_NUM_COUNTERS;i++) {
196 info->pmu_telem.host_counts[i] += snap[i] - info->pmu_telem.last_snapshot[i];
199 if (HAVE(V3_PMON_CLOCK_COUNT) && HAVE(V3_PMON_RETIRED_INST_COUNT)) {
200 update_ucpi_estimate(&(info->pmu_telem.host_ucpi_estimate), info->pmu_telem.host_counts, info->pmu_telem.last_snapshot);
202 if (HAVE(V3_PMON_CACHE_MISS_COUNT) && HAVE(V3_PMON_MEM_LOAD_COUNT)) {
203 update_umpl_estimate(&(info->pmu_telem.host_umpl_estimate), info->pmu_telem.host_counts, info->pmu_telem.last_snapshot);
206 for (i=0;i<PMU_NUM_COUNTERS;i++) {
207 info->pmu_telem.last_snapshot[i] = snap[i];
210 info->pmu_telem.state = AWAIT_EXIT;
215 PrintError(info->vm_info, info, "Impossible state in on pmu telemetry entry\n");
223 void v3_pmu_telemetry_exit(struct guest_info *info)
225 if (!info->vm_info->enable_telemetry) {
229 switch (info->pmu_telem.state) {
231 // AWAIT_EXIT - the snapshot in the struct is from the last entryx
232 uint64_t snap[PMU_NUM_COUNTERS];
237 for (i=0;i<PMU_NUM_COUNTERS;i++) {
238 info->pmu_telem.guest_counts[i] += snap[i] - info->pmu_telem.last_snapshot[i];
241 if (HAVE(V3_PMON_CLOCK_COUNT) && HAVE(V3_PMON_RETIRED_INST_COUNT)) {
242 update_ucpi_estimate(&(info->pmu_telem.guest_ucpi_estimate), info->pmu_telem.guest_counts, info->pmu_telem.last_snapshot);
244 if (HAVE(V3_PMON_CACHE_MISS_COUNT) && HAVE(V3_PMON_MEM_LOAD_COUNT)) {
245 update_umpl_estimate(&(info->pmu_telem.guest_umpl_estimate), info->pmu_telem.guest_counts, info->pmu_telem.last_snapshot);
248 for (i=0;i<PMU_NUM_COUNTERS;i++) {
249 info->pmu_telem.last_snapshot[i] = snap[i];
252 info->pmu_telem.state = AWAIT_ENTRY;
256 PrintError(info->vm_info, info, "Impossible state in on pmu telemetry exit\n");
262 void v3_pmu_telemetry_end(struct guest_info *info)
264 if (!info->vm_info->enable_telemetry) {
268 STOP(V3_PMON_RETIRED_INST_COUNT);
269 STOP(V3_PMON_CLOCK_COUNT);
270 STOP(V3_PMON_MEM_LOAD_COUNT);
271 STOP(V3_PMON_MEM_STORE_COUNT);
272 STOP(V3_PMON_CACHE_MISS_COUNT);
273 STOP(V3_PMON_TLB_MISS_COUNT);
277 info->pmu_telem.state=AWAIT_FIRST_ENTRY;
279 // Umm.... there is no v3_remove_telemtry_cb ? WTF?