Split telemetry into global and per-core to avoid race printing core telemetry

diff --cc palacios/include/palacios/vmm_telemetry.h
index 0677e2d,0677e2d..2fb5bc2
--- 1/palacios/include/palacios/vmm_telemetry.h
--- 2/palacios/include/palacios/vmm_telemetry.h
+++ b/palacios/include/palacios/vmm_telemetry.h
@@@ -59,7 -59,7 +59,8 @@@ void v3_deinit_core_telemetry(struct gu
  void v3_telemetry_start_exit(struct guest_info * info);
  void v3_telemetry_end_exit(struct guest_info * info, uint_t exit_code);
  
--void v3_print_telemetry(struct v3_vm_info * vm);
++void v3_print_global_telemetry(struct v3_vm_info * vm);
++void v3_print_core_telemetry(struct guest_info * vm);
  
  
  void v3_add_telemetry_cb(struct v3_vm_info * vm, 

diff --cc palacios/src/devices/apic.c
index 6abeeb9,62fbb4d..d88852d
--- 1/palacios/src/devices/apic.c
--- 2/palacios/src/devices/apic.c
+++ b/palacios/src/devices/apic.c
@@@ -1432,8 -1432,8 +1432,7 @@@ static int apic_write(struct guest_inf
        }
        case INT_CMD_HI_OFFSET: {
            apic->int_cmd.hi = op_val;
-           PrintDebug("apic %u: core %u: writing command high=0x%x\n", apic->lapic_id.val, core->vcpu_id,apic->int_cmd.hi);
- 
+           //V3_Print("apic %u: core %u: writing command high=0x%x\n", apic->lapic_id.val, core->vcpu_id,apic->int_cmd.hi);
 -
            break;
        }
        // Unhandled Registers

diff --cc palacios/src/devices/keyboard.c
index 1dc482d,1dc482d..794e8b3
--- 1/palacios/src/devices/keyboard.c
--- 2/palacios/src/devices/keyboard.c
+++ b/palacios/src/devices/keyboard.c
@@@ -388,7 -388,7 +388,7 @@@ static int key_event_handler(struct v3_
  #ifdef V3_CONFIG_TELEMETRY
  
      else if (evt->scan_code == 0x41) { // F7 telemetry dump
--      v3_print_telemetry(vm);
++      v3_print_global_telemetry(vm);
      } 
  #endif
  #ifdef V3_CONFIG_SYMMOD

diff --cc palacios/src/palacios/vmm_telemetry.c
index 42e7822,42e7822..4ddea67
--- 1/palacios/src/palacios/vmm_telemetry.c
--- 2/palacios/src/palacios/vmm_telemetry.c
+++ b/palacios/src/palacios/vmm_telemetry.c
@@@ -204,7 -204,7 +204,8 @@@ void v3_telemetry_end_exit(struct guest
  
      // check if the exit count has expired
      if ((telemetry->exit_cnt % telemetry->vm_telem->granularity) == 0) {
--      v3_print_telemetry(info->vm_info);
++      v3_print_global_telemetry(info->vm_info);
++      v3_print_core_telemetry(info);
      }
  }
  
@@@ -233,11 -233,11 +234,35 @@@ static int free_callback(struct v3_vm_i
  }
  
  
--void v3_print_telemetry(struct v3_vm_info * vm) {
++void v3_print_core_telemetry(struct guest_info * core ) {
++    struct exit_event * evt = NULL;
++    struct rb_node * node = v3_rb_first(&(core->core_telem.exit_root));
++    
++    V3_Print("Exit information for Core %d\n", core->vcpu_id);
++    
++    if (!node) { 
++      V3_Print("No information yet for this core\n");
++      return;
++    }
++
++    do {
++       evt = rb_entry(node, struct exit_event, tree_node);
++       const char * code_str = vmexit_code_to_str(evt->exit_code);
++          
++       V3_Print("%s:%sCnt=%u,%sAvg. Time=%u\n", 
++                code_str,
++                (strlen(code_str) > 13) ? "\t" : "\t\t",
++                evt->cnt,
++                (evt->cnt >= 100) ? "\t" : "\t\t",
++                (uint32_t)(evt->handler_time / evt->cnt));
++    } while ((node = v3_rb_next(node)));
++    return;
++}
++
++void v3_print_global_telemetry(struct v3_vm_info * vm) {
      struct v3_telemetry_state * telemetry = &(vm->telemetry);
      uint64_t invoke_tsc = 0;
      char hdr_buf[32];
--    int i;
  
      rdtscll(invoke_tsc);
  
@@@ -245,35 -245,35 +270,6 @@@
  
      V3_Print("%stelemetry window tsc cnt: %d\n", hdr_buf, (uint32_t)(invoke_tsc - telemetry->prev_tsc));
  
--
--    // Exit Telemetry
--    for (i = 0; i < vm->num_cores; i++) {
--      struct guest_info * core = &(vm->cores[i]);
--      struct exit_event * evt = NULL;
--      struct rb_node * node = v3_rb_first(&(core->core_telem.exit_root));
--      
--      V3_Print("Exit information for Core %d\n", core->vcpu_id);
--
--      if (!node) { 
--          V3_Print("No information yet for this core\n");
--          continue;
--      }
--
--      do {
--          evt = rb_entry(node, struct exit_event, tree_node);
--          const char * code_str = vmexit_code_to_str(evt->exit_code);
--          
--          V3_Print("%s%s:%sCnt=%u,%sAvg. Time=%u\n", 
--                   hdr_buf,
--                   code_str,
--                   (strlen(code_str) > 13) ? "\t" : "\t\t",
--                   evt->cnt,
--                   (evt->cnt >= 100) ? "\t" : "\t\t",
--                   (uint32_t)(evt->handler_time / evt->cnt));
--      } while ((node = v3_rb_next(node)));
--    }
--
--
      // Registered callbacks
      {
        struct telemetry_cb * cb = NULL;
@@@ -286,5 -286,5 +282,4 @@@
      telemetry->prev_tsc = invoke_tsc;
  
      V3_Print("%s Telemetry done\n", hdr_buf);
--
  }