2 * This file is part of the Palacios Virtual Machine Monitor developed
3 * by the V3VEE Project with funding from the United States National
4 * Science Foundation and the Department of Energy.
6 * The V3VEE Project is a joint project between Northwestern University
7 * and the University of New Mexico. You can find out more at
10 * Copyright (c) 2013, The V3VEE Project <http://www.v3vee.org>
11 * All rights reserved.
13 * Author: Chang S. Bae <chang.bae@eecs.northwestern.edu>
14 * Peter Dinda <pdinda@northwestern.edu>
16 * This is free software. You are permitted to use,
17 * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
19 #include <palacios/vm_guest.h>
20 #include <palacios/vmm_telemetry.h>
21 #include <palacios/vmm_pmu_telemetry.h>
22 #include <palacios/vmm_sprintf.h>
27 V3_PMON_RETIRED_INST_COUNT,
29 V3_PMON_MEM_LOAD_COUNT,
30 V3_PMON_MEM_STORE_COUNT,
31 V3_PMON_CACHE_MISS_COUNT,
32 V3_PMON_TLB_MISS_COUNT
37 cache Misses per instruction
40 #define HAVE(WHAT) (info->pmu_telem.active_counters[WHAT])
42 #define GUEST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pmu_telem.guest_counts[WHAT]); } } while (0)
43 #define HOST(WHAT) do { if (HAVE(WHAT)) { V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, #WHAT, info->pmu_telem.host_counts[WHAT]); } } while (0)
46 static int print_pmu_data(struct guest_info *info, char * hdr)
48 GUEST(V3_PMON_RETIRED_INST_COUNT);
49 GUEST(V3_PMON_CLOCK_COUNT);
50 GUEST(V3_PMON_MEM_LOAD_COUNT);
51 GUEST(V3_PMON_MEM_STORE_COUNT);
52 GUEST(V3_PMON_CACHE_MISS_COUNT);
53 GUEST(V3_PMON_TLB_MISS_COUNT);
54 V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UCPI",info->pmu_telem.guest_ucpi_estimate);
55 V3_Print(info->vm_info, info, "%sGUEST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UMPL", info->pmu_telem.guest_umpl_estimate);
57 HOST(V3_PMON_RETIRED_INST_COUNT);
58 HOST(V3_PMON_CLOCK_COUNT);
59 HOST(V3_PMON_MEM_LOAD_COUNT);
60 HOST(V3_PMON_MEM_STORE_COUNT);
61 HOST(V3_PMON_CACHE_MISS_COUNT);
62 HOST(V3_PMON_TLB_MISS_COUNT);
63 V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UCPI",info->pmu_telem.host_ucpi_estimate);
64 V3_Print(info->vm_info, info, "%sHOST:%u:%u:%s = %llu\n", hdr, info->vcpu_id, info->pcpu_id, "UMPL", info->pmu_telem.host_umpl_estimate);
70 static void telemetry_pmu(struct v3_vm_info * vm, void * private_data, char * hdr)
73 struct guest_info *core = NULL;
76 * work through each pcore (vcore for now per excluding oversubscription) and gathering info
78 for(i=0; i<vm->num_cores; i++) {
79 core = &(vm->cores[i]);
80 if((core->core_run_state != CORE_RUNNING)) continue;
81 print_pmu_data(core, hdr);
88 if(v3_pmu_start_tracking(WHAT) == -1) { \
89 PrintError(info->vm_info, info, "Failed to start tracking of %s\n", #WHAT); \
90 info->pmu_telem.active_counters[WHAT]=0; \
92 info->pmu_telem.active_counters[WHAT]=1;\
98 if(info->pmu_telem.active_counters[WHAT]) { \
99 if (v3_pmu_stop_tracking(WHAT) == -1) { \
100 PrintError(info->vm_info, info, "Failed to stop tracking of %s\n", #WHAT); \
102 info->pmu_telem.active_counters[WHAT]=0; \
107 void v3_pmu_telemetry_start(struct guest_info *info)
109 if (!info->vm_info->enable_telemetry) {
113 memset(&(info->pmu_telem),0,sizeof(struct v3_core_pmu_telemetry));
117 START(V3_PMON_RETIRED_INST_COUNT);
118 START(V3_PMON_CLOCK_COUNT);
119 START(V3_PMON_MEM_LOAD_COUNT);
120 START(V3_PMON_MEM_STORE_COUNT);
121 START(V3_PMON_CACHE_MISS_COUNT);
122 START(V3_PMON_TLB_MISS_COUNT);
125 info->pmu_telem.state=AWAIT_FIRST_ENTRY;
128 if (info->vcpu_id==0) {
129 v3_add_telemetry_cb(info->vm_info, telemetry_pmu, NULL);
134 static void inline snapshot(uint64_t vals[]) {
135 vals[V3_PMON_RETIRED_INST_COUNT] = v3_pmu_get_value(V3_PMON_RETIRED_INST_COUNT);
136 vals[V3_PMON_CLOCK_COUNT] = v3_pmu_get_value(V3_PMON_CLOCK_COUNT);
137 vals[V3_PMON_MEM_LOAD_COUNT] = v3_pmu_get_value(V3_PMON_MEM_LOAD_COUNT);
138 vals[V3_PMON_MEM_STORE_COUNT] = v3_pmu_get_value(V3_PMON_MEM_STORE_COUNT);
139 vals[V3_PMON_CACHE_MISS_COUNT] = v3_pmu_get_value(V3_PMON_CACHE_MISS_COUNT);
140 vals[V3_PMON_TLB_MISS_COUNT] = v3_pmu_get_value(V3_PMON_TLB_MISS_COUNT);
144 #define ALPHA_DENOM 8 // we are counting in 8ths
145 #define ALPHA_NUM 1 // 1/8 to estimate
146 #define OM_ALPHA_NUM 7 // 7/8 to the update
148 static inline void update_ucpi_estimate(uint64_t *estimate, uint64_t cur[], uint64_t last[])
150 // 1e6 times the number of cycles since last
151 uint64_t ucycles = 1000000 * (cur[V3_PMON_CLOCK_COUNT] - last[V3_PMON_CLOCK_COUNT]);
152 uint64_t insts = cur[V3_PMON_RETIRED_INST_COUNT] - last[V3_PMON_RETIRED_INST_COUNT];
159 *estimate = ((ALPHA_NUM * (*estimate)) + (OM_ALPHA_NUM * ((ucycles/insts)))) / ALPHA_DENOM;
163 static inline void update_umpl_estimate(uint64_t *estimate, uint64_t cur[], uint64_t last[])
165 // 1e6 times the number of misses since the last time
166 uint64_t umisses = 1000000 * (cur[V3_PMON_CACHE_MISS_COUNT] - last[V3_PMON_CACHE_MISS_COUNT]);
167 uint64_t loads = cur[V3_PMON_MEM_LOAD_COUNT] - last[V3_PMON_MEM_LOAD_COUNT];
173 *estimate = ((ALPHA_NUM * (*estimate)) + (OM_ALPHA_NUM * ((umisses/loads)))) / ALPHA_DENOM;
177 void v3_pmu_telemetry_enter(struct guest_info *info)
179 if (!info->vm_info->enable_telemetry) {
183 switch (info->pmu_telem.state) {
184 case AWAIT_FIRST_ENTRY:
185 snapshot(info->pmu_telem.last_snapshot);
186 info->pmu_telem.state=AWAIT_EXIT;
190 // AWAIT_ENTRY - the snapshot in the struct is from the last exit
191 uint64_t snap[PMU_NUM_COUNTERS];
196 for (i=0;i<PMU_NUM_COUNTERS;i++) {
197 info->pmu_telem.host_counts[i] += snap[i] - info->pmu_telem.last_snapshot[i];
200 if (HAVE(V3_PMON_CLOCK_COUNT) && HAVE(V3_PMON_RETIRED_INST_COUNT)) {
201 update_ucpi_estimate(&(info->pmu_telem.host_ucpi_estimate), snap, info->pmu_telem.last_snapshot);
203 if (HAVE(V3_PMON_CACHE_MISS_COUNT) && HAVE(V3_PMON_MEM_LOAD_COUNT)) {
204 update_umpl_estimate(&(info->pmu_telem.host_umpl_estimate), snap, info->pmu_telem.last_snapshot);
207 for (i=0;i<PMU_NUM_COUNTERS;i++) {
208 info->pmu_telem.last_snapshot[i] = snap[i];
211 info->pmu_telem.state = AWAIT_EXIT;
216 PrintError(info->vm_info, info, "Impossible state in on pmu telemetry entry\n");
224 void v3_pmu_telemetry_exit(struct guest_info *info)
226 if (!info->vm_info->enable_telemetry) {
230 switch (info->pmu_telem.state) {
232 // AWAIT_EXIT - the snapshot in the struct is from the last entryx
233 uint64_t snap[PMU_NUM_COUNTERS];
238 for (i=0;i<PMU_NUM_COUNTERS;i++) {
239 info->pmu_telem.guest_counts[i] += snap[i] - info->pmu_telem.last_snapshot[i];
242 if (HAVE(V3_PMON_CLOCK_COUNT) && HAVE(V3_PMON_RETIRED_INST_COUNT)) {
243 update_ucpi_estimate(&(info->pmu_telem.guest_ucpi_estimate), snap, info->pmu_telem.last_snapshot);
246 if (HAVE(V3_PMON_CACHE_MISS_COUNT) && HAVE(V3_PMON_MEM_LOAD_COUNT)) {
247 update_umpl_estimate(&(info->pmu_telem.guest_umpl_estimate), snap, info->pmu_telem.last_snapshot);
250 for (i=0;i<PMU_NUM_COUNTERS;i++) {
251 info->pmu_telem.last_snapshot[i] = snap[i];
254 info->pmu_telem.state = AWAIT_ENTRY;
258 PrintError(info->vm_info, info, "Impossible state in on pmu telemetry exit\n");
264 void v3_pmu_telemetry_end(struct guest_info *info)
266 if (!info->vm_info->enable_telemetry) {
270 STOP(V3_PMON_RETIRED_INST_COUNT);
271 STOP(V3_PMON_CLOCK_COUNT);
272 STOP(V3_PMON_MEM_LOAD_COUNT);
273 STOP(V3_PMON_MEM_STORE_COUNT);
274 STOP(V3_PMON_CACHE_MISS_COUNT);
275 STOP(V3_PMON_TLB_MISS_COUNT);
279 info->pmu_telem.state=AWAIT_FIRST_ENTRY;
281 // Umm.... there is no v3_remove_telemtry_cb ? WTF?