Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


VM Reset Bugfixes
[palacios.git] / palacios / src / palacios / svm.c
index e6ded68..a007201 100644 (file)
  * All rights reserved.
  *
  * Author: Jack Lange <jarusl@cs.northwestern.edu>
+ *         Peter Dinda <jarusl@cs.northwestern.edu> (Reset)
  *
  * This is free software.  You are permitted to use,
  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  */
 
 
+
 #include <palacios/svm.h>
 #include <palacios/vmm.h>
 
 
 #include <palacios/vmm_rbtree.h>
 #include <palacios/vmm_barrier.h>
+#include <palacios/vmm_debug.h>
+
+#include <palacios/vmm_perftune.h>
+
+#include <palacios/vmm_bios.h>
+
 
 #ifdef V3_CONFIG_CHECKPOINT
 #include <palacios/vmm_checkpoint.h>
 
 #include <palacios/vmm_sprintf.h>
 
+#ifdef V3_CONFIG_MEM_TRACK
+#include <palacios/vmm_mem_track.h>
+#endif 
+
+#ifdef V3_CONFIG_TM_FUNC
+#include <extensions/trans_mem.h>
+#endif
 
 #ifndef V3_CONFIG_DEBUG_SVM
 #undef PrintDebug
 #endif
 
 
+
 uint32_t v3_last_exit;
 
 // This is a global pointer to the host's VMCB
+// These are physical addresses
 static addr_t host_vmcbs[V3_CONFIG_MAX_CPUS] = { [0 ... V3_CONFIG_MAX_CPUS - 1] = 0};
 
 
@@ -68,12 +85,13 @@ extern void v3_clgi();
 extern int v3_svm_launch(vmcb_t * vmcb, struct v3_gprs * vm_regs, vmcb_t * host_vmcb);
 
 
+
 static vmcb_t * Allocate_VMCB() {
     vmcb_t * vmcb_page = NULL;
-    addr_t vmcb_pa = (addr_t)V3_AllocPages(1);
+    addr_t vmcb_pa = (addr_t)V3_AllocPages(1);   // need not be shadow safe, not exposed to guest
 
     if ((void *)vmcb_pa == NULL) {
-       PrintError("Error allocating VMCB\n");
+      PrintError(VM_NONE, VCORE_NONE, "Error allocating VMCB\n");
        return NULL;
     }
 
@@ -104,59 +122,68 @@ static int v3_svm_handle_efer_write(struct guest_info * core, uint_t msr, struct
     return 0;
 }
 
+/*
+ * This is invoked both on an initial boot and on a reset
+ * 
+ * The difference is that on a reset we will not rehook anything
+ *
+ */
 
 static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
     vmcb_ctrl_t * ctrl_area = GET_VMCB_CTRL_AREA(vmcb);
     vmcb_saved_state_t * guest_state = GET_VMCB_SAVE_STATE_AREA(vmcb);
     uint_t i;
 
+    if (core->core_run_state!=CORE_INVALID && core->core_run_state!=CORE_RESETTING) { 
+       PrintError(core->vm_info, core, "Atempt to Init_VMCB_BIOS in invalid state (%d)\n",core->core_run_state);
+       return;
+    }
+
+    // need to invalidate any shadow page tables early
+    if (core->shdw_pg_mode == SHADOW_PAGING && core->core_run_state==CORE_RESETTING) {
+       if (v3_get_vm_cpu_mode(core) != REAL) {
+           if (v3_invalidate_shadow_pts(core) == -1) {
+               PrintError(core->vm_info,core,"Could not invalidate shadow page tables\n");
+               return;
+           }
+       }
+    }
+
+    // Guarantee we are starting from a clean slate
+    // even on a reset
+    memset(vmcb,0,4096);
 
-    //
     ctrl_area->svm_instrs.VMRUN = 1;
     ctrl_area->svm_instrs.VMMCALL = 1;
     ctrl_area->svm_instrs.VMLOAD = 1;
     ctrl_area->svm_instrs.VMSAVE = 1;
     ctrl_area->svm_instrs.STGI = 1;
     ctrl_area->svm_instrs.CLGI = 1;
-    ctrl_area->svm_instrs.SKINIT = 1;
-    ctrl_area->svm_instrs.ICEBP = 1;
-    ctrl_area->svm_instrs.WBINVD = 1;
+    ctrl_area->svm_instrs.SKINIT = 1; // secure startup... why
+    ctrl_area->svm_instrs.ICEBP = 1;  // in circuit emulator breakpoint
+    ctrl_area->svm_instrs.WBINVD = 1; // write back and invalidate caches... why?
     ctrl_area->svm_instrs.MONITOR = 1;
     ctrl_area->svm_instrs.MWAIT_always = 1;
     ctrl_area->svm_instrs.MWAIT_if_armed = 1;
-    ctrl_area->instrs.INVLPGA = 1;
+    ctrl_area->instrs.INVLPGA = 1;   // invalidate page in asid... why?
     ctrl_area->instrs.CPUID = 1;
 
     ctrl_area->instrs.HLT = 1;
 
-#ifdef V3_CONFIG_TIME_VIRTUALIZE_TSC
-    ctrl_area->instrs.RDTSC = 1;
-    ctrl_area->svm_instrs.RDTSCP = 1;
-#endif
+    /* Set at VMM launch as needed */
+    ctrl_area->instrs.RDTSC = 0;
+    ctrl_area->svm_instrs.RDTSCP = 0;
 
-    // guest_state->cr0 = 0x00000001;    // PE 
-  
-    /*
-      ctrl_area->exceptions.de = 1;
-      ctrl_area->exceptions.df = 1;
-      
-      ctrl_area->exceptions.ts = 1;
-      ctrl_area->exceptions.ss = 1;
-      ctrl_area->exceptions.ac = 1;
-      ctrl_area->exceptions.mc = 1;
-      ctrl_area->exceptions.gp = 1;
-      ctrl_area->exceptions.ud = 1;
-      ctrl_area->exceptions.np = 1;
-      ctrl_area->exceptions.of = 1;
-      
-      ctrl_area->exceptions.nmi = 1;
-    */
+
+#ifdef V3_CONFIG_TM_FUNC
+    v3_tm_set_excp_intercepts(ctrl_area);
+#endif
     
 
     ctrl_area->instrs.NMI = 1;
     ctrl_area->instrs.SMI = 0; // allow SMIs to run in guest
     ctrl_area->instrs.INIT = 1;
-    ctrl_area->instrs.PAUSE = 1;
+    //    ctrl_area->instrs.PAUSE = 1;    // do not care as does not halt
     ctrl_area->instrs.shutdown_evts = 1;
 
 
@@ -166,30 +193,50 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
 
     /* Setup Guest Machine state */
 
-    core->vm_regs.rsp = 0x00;
-    core->rip = 0xfff0;
+    memset(&core->vm_regs,0,sizeof(core->vm_regs));
+    memset(&core->ctrl_regs,0,sizeof(core->ctrl_regs));
+    memset(&core->dbg_regs,0,sizeof(core->dbg_regs));
+    memset(&core->segments,0,sizeof(core->segments));    
+    memset(&core->msrs,0,sizeof(core->msrs));    
+    memset(&core->fp_state,0,sizeof(core->fp_state));    
 
-    core->vm_regs.rdx = 0x00000f00;
+    // reset interrupts
+    core->intr_core_state.irq_pending=0; 
+    core->intr_core_state.irq_started=0; 
+    core->intr_core_state.swintr_posted=0; 
 
+    // reset exceptions
+    core->excp_state.excp_pending=0;
 
+    // reset of gprs to expected values at init
+    core->vm_regs.rsp = 0x00;
+    core->rip = 0xfff0;
+    core->vm_regs.rdx = 0x00000f00;  // family/stepping/etc
+
+    
     core->cpl = 0;
 
     core->ctrl_regs.rflags = 0x00000002; // The reserved bit is always 1
-    core->ctrl_regs.cr0 = 0x60010010; // Set the WP flag so the memory hooks work in real-mode
-    core->ctrl_regs.efer |= EFER_MSR_svm_enable;
-
 
+    core->ctrl_regs.cr0 = 0x60010010; // Set the WP flag so the memory hooks work in real-mode
+    core->shdw_pg_state.guest_cr0 = core->ctrl_regs.cr0;
 
+    // cr3 zeroed above
+    core->shdw_pg_state.guest_cr3 = core->ctrl_regs.cr3;
+    // cr4 zeroed above
+    core->shdw_pg_state.guest_cr4 = core->ctrl_regs.cr4;
 
+    core->ctrl_regs.efer |= EFER_MSR_svm_enable ;
+    core->shdw_pg_state.guest_efer.value = core->ctrl_regs.efer;
 
     core->segments.cs.selector = 0xf000;
     core->segments.cs.limit = 0xffff;
-    core->segments.cs.base = 0x0000000f0000LL;
+    core->segments.cs.base = 0x0000f0000LL;
 
     // (raw attributes = 0xf3)
-    core->segments.cs.type = 0x3;
+    core->segments.cs.type = 0xa;
     core->segments.cs.system = 0x1;
-    core->segments.cs.dpl = 0x3;
+    core->segments.cs.dpl = 0x0;
     core->segments.cs.present = 1;
 
 
@@ -204,27 +251,35 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
        seg->selector = 0x0000;
        //    seg->base = seg->selector << 4;
        seg->base = 0x00000000;
-       seg->limit = ~0u;
+       seg->limit = 0xffff;
 
        // (raw attributes = 0xf3)
-       seg->type = 0x3;
+       seg->type = 0x2;
        seg->system = 0x1;
-       seg->dpl = 0x3;
+       seg->dpl = 0x0;
        seg->present = 1;
     }
 
+    core->segments.gdtr.selector = 0x0000;
     core->segments.gdtr.limit = 0x0000ffff;
     core->segments.gdtr.base = 0x0000000000000000LL;
+    core->segments.gdtr.dpl = 0x0;
+
+    core->segments.idtr.selector = 0x0000; 
     core->segments.idtr.limit = 0x0000ffff;
     core->segments.idtr.base = 0x0000000000000000LL;
-
-    core->segments.ldtr.selector = 0x0000;
     core->segments.ldtr.limit = 0x0000ffff;
     core->segments.ldtr.base = 0x0000000000000000LL;
+    core->segments.ldtr.system = 0;
+    core->segments.ldtr.type = 0x2;
+    core->segments.ldtr.dpl = 0x0;
+
     core->segments.tr.selector = 0x0000;
     core->segments.tr.limit = 0x0000ffff;
     core->segments.tr.base = 0x0000000000000000LL;
-
+    core->segments.tr.system = 0;
+    core->segments.tr.type = 0x3;
+    core->segments.tr.dpl = 0x0;
 
     core->dbg_regs.dr6 = 0x00000000ffff0ff0LL;
     core->dbg_regs.dr7 = 0x0000000000000400LL;
@@ -237,46 +292,63 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
     ctrl_area->instrs.MSR_PROT = 1;   
 
 
-    PrintDebug("Exiting on interrupts\n");
     ctrl_area->guest_ctrl.V_INTR_MASKING = 1;
     ctrl_area->instrs.INTR = 1;
+    // The above also assures the TPR changes (CR8) are only virtual
 
 
-    v3_hook_msr(core->vm_info, EFER_MSR, 
-               &v3_handle_efer_read,
-               &v3_svm_handle_efer_write, 
-               core);
+    // However, we need to see TPR writes since they will
+    // affect the virtual apic
+    // we reflect out cr8 to ctrl_regs->apic_tpr
+    ctrl_area->cr_reads.cr8 = 1;
+    ctrl_area->cr_writes.cr8 = 1;
+    // We will do all TPR comparisons in the virtual apic
+    // We also do not want the V_TPR to be able to mask the PIC
+    ctrl_area->guest_ctrl.V_IGN_TPR = 1;
+
+    
+
+    if (core->core_run_state == CORE_INVALID) { 
+       v3_hook_msr(core->vm_info, EFER_MSR, 
+                   &v3_handle_efer_read,
+                   &v3_svm_handle_efer_write, 
+                   core);
+    }
 
     if (core->shdw_pg_mode == SHADOW_PAGING) {
-       PrintDebug("Creating initial shadow page table\n");
        
        /* JRL: This is a performance killer, and a simplistic solution */
        /* We need to fix this */
        ctrl_area->TLB_CONTROL = 1;
        ctrl_area->guest_ASID = 1;
        
-       
-       if (v3_init_passthrough_pts(core) == -1) {
-           PrintError("Could not initialize passthrough page tables\n");
-           return ;
-       }
 
+       if (core->core_run_state == CORE_INVALID) { 
+           if (v3_init_passthrough_pts(core) == -1) {
+               PrintError(core->vm_info, core, "Could not initialize passthrough page tables\n");
+               return ;
+           }
+           // the shadow page tables are OK since we have not initialized hem yet
+       } else {
+           // CORE_RESETTING
+           // invalidation of shadow page tables happened earlier in this function
+       }
 
        core->shdw_pg_state.guest_cr0 = 0x0000000000000010LL;
-       PrintDebug("Created\n");
        
        core->ctrl_regs.cr0 |= 0x80000000;
-       core->ctrl_regs.cr3 = core->direct_map_pt;
+
+        v3_activate_passthrough_pt(core);
 
        ctrl_area->cr_reads.cr0 = 1;
        ctrl_area->cr_writes.cr0 = 1;
-       //ctrl_area->cr_reads.cr4 = 1;
+       //intercept cr4 read so shadow pager can use PAE independently of guest
+       ctrl_area->cr_reads.cr4 = 1;
        ctrl_area->cr_writes.cr4 = 1;
        ctrl_area->cr_reads.cr3 = 1;
        ctrl_area->cr_writes.cr3 = 1;
 
 
-
        ctrl_area->instrs.INVLPG = 1;
 
        ctrl_area->exceptions.pf = 1;
@@ -284,7 +356,6 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
        guest_state->g_pat = 0x7040600070406ULL;
 
 
-
     } else if (core->shdw_pg_mode == NESTED_PAGING) {
        // Flush the TLB on entries/exits
        ctrl_area->TLB_CONTROL = 1;
@@ -293,12 +364,16 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
        // Enable Nested Paging
        ctrl_area->NP_ENABLE = 1;
 
-       PrintDebug("NP_Enable at 0x%p\n", (void *)&(ctrl_area->NP_ENABLE));
+       PrintDebug(core->vm_info, core, "NP_Enable at 0x%p\n", (void *)&(ctrl_area->NP_ENABLE));
 
        // Set the Nested Page Table pointer
-       if (v3_init_passthrough_pts(core) == -1) {
-           PrintError("Could not initialize Nested page tables\n");
-           return ;
+       if (core->core_run_state == CORE_INVALID) { 
+           if (v3_init_passthrough_pts(core) == -1) {
+               PrintError(core->vm_info, core, "Could not initialize Nested page tables\n");
+               return ;
+           }
+       } else {
+           // the existing nested page tables will work fine
        }
 
        ctrl_area->N_CR3 = core->direct_map_pt;
@@ -307,13 +382,14 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
     }
     
     /* tell the guest that we don't support SVM */
-    v3_hook_msr(core->vm_info, SVM_VM_CR_MSR, 
-       &v3_handle_vm_cr_read,
-       &v3_handle_vm_cr_write, 
-       core);
-
+    if (core->core_run_state == CORE_INVALID) { 
+       v3_hook_msr(core->vm_info, SVM_VM_CR_MSR, 
+                   &v3_handle_vm_cr_read,
+                   &v3_handle_vm_cr_write, 
+                   core);
+    }
 
-    {
+    if (core->core_run_state == CORE_INVALID) { 
 #define INT_PENDING_AMD_MSR            0xc0010055
 
        v3_hook_msr(core->vm_info, IA32_STAR_MSR, NULL, NULL, NULL);
@@ -333,55 +409,180 @@ static void Init_VMCB_BIOS(vmcb_t * vmcb, struct guest_info * core) {
        // Passthrough read operations are ok.
        v3_hook_msr(core->vm_info, INT_PENDING_AMD_MSR, NULL, v3_msr_unhandled_write, NULL);
     }
+
+
 }
 
 
 int v3_init_svm_vmcb(struct guest_info * core, v3_vm_class_t vm_class) {
 
-    PrintDebug("Allocating VMCB\n");
+    PrintDebug(core->vm_info, core, "Allocating VMCB\n");
     core->vmm_data = (void *)Allocate_VMCB();
     
     if (core->vmm_data == NULL) {
-       PrintError("Could not allocate VMCB, Exiting...\n");
+       PrintError(core->vm_info, core, "Could not allocate VMCB, Exiting...\n");
        return -1;
     }
 
     if (vm_class == V3_PC_VM) {
-       PrintDebug("Initializing VMCB (addr=%p)\n", (void *)core->vmm_data);
+       PrintDebug(core->vm_info, core, "Initializing VMCB (addr=%p)\n", (void *)core->vmm_data);
        Init_VMCB_BIOS((vmcb_t*)(core->vmm_data), core);
     } else {
-       PrintError("Invalid VM class\n");
+       PrintError(core->vm_info, core, "Invalid VM class\n");
        return -1;
     }
 
+    core->core_run_state = CORE_STOPPED;
+
     return 0;
 }
 
 
 int v3_deinit_svm_vmcb(struct guest_info * core) {
-    V3_FreePages(V3_PAddr(core->vmm_data), 1);
+    if (core->vmm_data) { 
+       V3_FreePages(V3_PAddr(core->vmm_data), 1);
+    }
     return 0;
 }
 
 
+static int svm_handle_standard_reset(struct guest_info *core)
+{
+    if (core->core_run_state != CORE_RESETTING) { 
+       return 0;
+    }
+
+    PrintDebug(core->vm_info,core,"Handling standard reset (guest state before follows)\n");
+
+#ifdef V3_CONFIG_DEBUG_SVM
+    v3_print_guest_state(core);
+#endif
+
+    // wait until all resetting cores get here (ROS or whole VM)
+    v3_counting_barrier(&core->vm_info->reset_barrier);
+
+    // I could be a ROS core, or I could be in a non-HVM 
+    // either way, if I'm core 0, I'm the leader
+    if (core->vcpu_id==0) {
+       uint64_t mem_size=core->vm_info->mem_size;
+
+#ifdef V3_CONFIG_HVM
+       // on a ROS reset, we should only 
+       // manipulate the part of the memory seen by
+       // the ROS
+       if (core->vm_info->hvm_state.is_hvm) { 
+           mem_size=v3_get_hvm_ros_memsize(core->vm_info);
+       }
+#endif
+       core->vm_info->run_state = VM_RESETTING;
+       // copy bioses again because some, 
+       // like seabios, assume
+       // this should also blow away the BDA and EBDA
+       PrintDebug(core->vm_info,core,"Clear memory (%p bytes)\n",(void*)core->vm_info->mem_size);
+       if (v3_set_gpa_memory(core, 0, mem_size, 0)!=mem_size) { 
+           PrintError(core->vm_info,core,"Clear of memory failed\n");
+       }
+       PrintDebug(core->vm_info,core,"Copying bioses\n");
+       if (v3_setup_bioses(core->vm_info, core->vm_info->cfg_data->cfg)) { 
+           PrintError(core->vm_info,core,"Setup of bioses failed\n");
+       }
+    }
+
+    Init_VMCB_BIOS((vmcb_t*)(core->vmm_data), core);
+
+    PrintDebug(core->vm_info,core,"InitVMCB done\n");
+
+    core->cpl = 0;
+    core->cpu_mode = REAL;
+    core->mem_mode = PHYSICAL_MEM;
+    //core->num_exits=0;
+
+    PrintDebug(core->vm_info,core,"Machine reset to REAL/PHYSICAL\n");
+
+    memset(V3_VAddr((void*)(host_vmcbs[V3_Get_CPU()])),0,4096*4); // good measure...
+
+    // core zero will be restarted by the main execution loop
+    core->core_run_state = CORE_STOPPED;
+
+    if (core->vcpu_id==0) { 
+       core->vm_info->run_state = VM_RUNNING;
+    } 
+
+#ifdef V3_CONFIG_DEBUG_SVM
+    PrintDebug(core->vm_info,core,"VMCB state at end of reset\n");
+    PrintDebugVMCB((vmcb_t*)(core->vmm_data));
+    PrintDebug(core->vm_info,core,"Guest state at end of reset\n");
+    v3_print_guest_state(core);
+#endif
+
+    // wait until we are all ready to go
+    v3_counting_barrier(&core->vm_info->reset_barrier);
+
+    PrintDebug(core->vm_info,core,"Returning with request for recycle loop\n");
+
+    return 1; // reboot is occuring
+
+}
+
 #ifdef V3_CONFIG_CHECKPOINT
 int v3_svm_save_core(struct guest_info * core, void * ctx){
 
-    v3_chkpt_save_8(ctx, "cpl", &(core->cpl));
-    v3_chkpt_save(ctx, "vmcb_data", PAGE_SIZE, core->vmm_data);
+  vmcb_saved_state_t * guest_area = GET_VMCB_SAVE_STATE_AREA(core->vmm_data); 
+
+  // Special case saves of data we need immediate access to
+  // in some cases
+  V3_CHKPT_SAVE(ctx, "CPL", core->cpl, failout);
+  V3_CHKPT_SAVE(ctx,"STAR", guest_area->star, failout); 
+  V3_CHKPT_SAVE(ctx,"CSTAR", guest_area->cstar, failout); 
+  V3_CHKPT_SAVE(ctx,"LSTAR", guest_area->lstar, failout); 
+  V3_CHKPT_SAVE(ctx,"SFMASK", guest_area->sfmask, failout); 
+  V3_CHKPT_SAVE(ctx,"KERNELGSBASE", guest_area->KernelGsBase, failout); 
+  V3_CHKPT_SAVE(ctx,"SYSENTER_CS", guest_area->sysenter_cs, failout); 
+  V3_CHKPT_SAVE(ctx,"SYSENTER_ESP", guest_area->sysenter_esp, failout); 
+  V3_CHKPT_SAVE(ctx,"SYSENTER_EIP", guest_area->sysenter_eip, failout); 
+  
+// and then we save the whole enchilada
+  if (v3_chkpt_save(ctx, "VMCB_DATA", PAGE_SIZE, core->vmm_data)) { 
+    PrintError(core->vm_info, core, "Could not save SVM vmcb\n");
+    goto failout;
+  }
+  
+  return 0;
+
+ failout:
+  PrintError(core->vm_info, core, "Failed to save SVM state for core\n");
+  return -1;
 
-    return 0;
 }
 
 int v3_svm_load_core(struct guest_info * core, void * ctx){
     
-    v3_chkpt_load_8(ctx, "cpl", &(core->cpl));
 
-    if (v3_chkpt_load(ctx, "vmcb_data", PAGE_SIZE, core->vmm_data) == -1) {
-       return -1;
-    }
+  vmcb_saved_state_t * guest_area = GET_VMCB_SAVE_STATE_AREA(core->vmm_data); 
+
+  // Reload what we special cased, which we will overwrite in a minute
+  V3_CHKPT_LOAD(ctx, "CPL", core->cpl, failout);
+  V3_CHKPT_LOAD(ctx,"STAR", guest_area->star, failout); 
+  V3_CHKPT_LOAD(ctx,"CSTAR", guest_area->cstar, failout); 
+  V3_CHKPT_LOAD(ctx,"LSTAR", guest_area->lstar, failout); 
+  V3_CHKPT_LOAD(ctx,"SFMASK", guest_area->sfmask, failout); 
+  V3_CHKPT_LOAD(ctx,"KERNELGSBASE", guest_area->KernelGsBase, failout); 
+  V3_CHKPT_LOAD(ctx,"SYSENTER_CS", guest_area->sysenter_cs, failout); 
+  V3_CHKPT_LOAD(ctx,"SYSENTER_ESP", guest_area->sysenter_esp, failout); 
+  V3_CHKPT_LOAD(ctx,"SYSENTER_EIP", guest_area->sysenter_eip, failout); 
+  
+  // and then we load the whole enchilada
+  if (v3_chkpt_load(ctx, "VMCB_DATA", PAGE_SIZE, core->vmm_data)) { 
+    PrintError(core->vm_info, core, "Could not load SVM vmcb\n");
+    goto failout;
+  }
+  
+  return 0;
+
+ failout:
+  PrintError(core->vm_info, core, "Failed to save SVM state for core\n");
+  return -1;
 
-    return 0;
 }
 #endif
 
@@ -394,7 +595,7 @@ static int update_irq_exit_state(struct guest_info * info) {
     if ((info->intr_core_state.irq_pending == 1) && (guest_ctrl->guest_ctrl.V_IRQ == 0)) {
        
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-       PrintDebug("INTAK cycle completed for irq %d\n", info->intr_core_state.irq_vector);
+       PrintDebug(info->vm_info, info, "INTAK cycle completed for irq %d\n", info->intr_core_state.irq_vector);
 #endif
 
        info->intr_core_state.irq_started = 1;
@@ -405,7 +606,7 @@ static int update_irq_exit_state(struct guest_info * info) {
 
     if ((info->intr_core_state.irq_started == 1) && (guest_ctrl->exit_int_info.valid == 0)) {
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-       PrintDebug("Interrupt %d taken by guest\n", info->intr_core_state.irq_vector);
+       PrintDebug(info->vm_info, info, "Interrupt %d taken by guest\n", info->intr_core_state.irq_vector);
 #endif
 
        // Interrupt was taken fully vectored
@@ -413,7 +614,7 @@ static int update_irq_exit_state(struct guest_info * info) {
 
     } else if ((info->intr_core_state.irq_started == 1) && (guest_ctrl->exit_int_info.valid == 1)) {
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-       PrintDebug("EXIT INT INFO is set (vec=%d)\n", guest_ctrl->exit_int_info.vector);
+       PrintDebug(info->vm_info, info, "EXIT INT INFO is set (vec=%d)\n", guest_ctrl->exit_int_info.vector);
 #endif
     }
 
@@ -439,7 +640,7 @@ static int update_irq_entry_state(struct guest_info * info) {
            guest_ctrl->EVENTINJ.error_code = info->excp_state.excp_error_code;
            guest_ctrl->EVENTINJ.ev = 1;
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-           PrintDebug("Injecting exception %d with error code %x\n", excp, guest_ctrl->EVENTINJ.error_code);
+           PrintDebug(info->vm_info, info, "Injecting exception %d with error code %x\n", excp, guest_ctrl->EVENTINJ.error_code);
 #endif
        }
        
@@ -448,7 +649,7 @@ static int update_irq_entry_state(struct guest_info * info) {
        guest_ctrl->EVENTINJ.valid = 1;
 
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-       PrintDebug("<%d> Injecting Exception %d (CR2=%p) (EIP=%p)\n", 
+       PrintDebug(info->vm_info, info, "<%d> Injecting Exception %d (CR2=%p) (EIP=%p)\n", 
                   (int)info->num_exits, 
                   guest_ctrl->EVENTINJ.vector, 
                   (void *)(addr_t)info->ctrl_regs.cr2,
@@ -458,42 +659,52 @@ static int update_irq_entry_state(struct guest_info * info) {
        v3_injecting_excp(info, excp);
     } else if (info->intr_core_state.irq_started == 1) {
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-       PrintDebug("IRQ pending from previous injection\n");
+       PrintDebug(info->vm_info, info, "IRQ pending from previous injection\n");
 #endif
        guest_ctrl->guest_ctrl.V_IRQ = 1;
        guest_ctrl->guest_ctrl.V_INTR_VECTOR = info->intr_core_state.irq_vector;
+
+       // We ignore the virtual TPR on this injection
+       // TPR/PPR tests have already been done in the APIC.
        guest_ctrl->guest_ctrl.V_IGN_TPR = 1;
-       guest_ctrl->guest_ctrl.V_INTR_PRIO = 0xf;
+       guest_ctrl->guest_ctrl.V_INTR_PRIO = info->intr_core_state.irq_vector >> 4 ;  // 0xf;
 
     } else {
        switch (v3_intr_pending(info)) {
            case V3_EXTERNAL_IRQ: {
-               uint32_t irq = v3_get_intr(info);
+               int irq = v3_get_intr(info); 
+
+               if (irq<0) {
+                 break;
+               }
 
                guest_ctrl->guest_ctrl.V_IRQ = 1;
                guest_ctrl->guest_ctrl.V_INTR_VECTOR = irq;
+
+               // We ignore the virtual TPR on this injection
+               // TPR/PPR tests have already been done in the APIC.
                guest_ctrl->guest_ctrl.V_IGN_TPR = 1;
-               guest_ctrl->guest_ctrl.V_INTR_PRIO = 0xf;
+               guest_ctrl->guest_ctrl.V_INTR_PRIO = info->intr_core_state.irq_vector >> 4 ;  // 0xf;
 
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-               PrintDebug("Injecting Interrupt %d (EIP=%p)\n", 
+               PrintDebug(info->vm_info, info, "Injecting Interrupt %d (EIP=%p)\n", 
                           guest_ctrl->guest_ctrl.V_INTR_VECTOR, 
                           (void *)(addr_t)info->rip);
 #endif
 
                info->intr_core_state.irq_pending = 1;
                info->intr_core_state.irq_vector = irq;
-               
+
                break;
+               
            }
            case V3_NMI:
                guest_ctrl->EVENTINJ.type = SVM_INJECTION_NMI;
                break;
            case V3_SOFTWARE_INTR:
                guest_ctrl->EVENTINJ.type = SVM_INJECTION_SOFT_INTR;
-
 #ifdef V3_CONFIG_DEBUG_INTERRUPTS
-               PrintDebug("Injecting software interrupt --  type: %d, vector: %d\n", 
+               PrintDebug(info->vm_info, info, "Injecting software interrupt --  type: %d, vector: %d\n", 
                           SVM_INJECTION_SOFT_INTR, info->intr_core_state.swintr_vector);
 #endif
                guest_ctrl->EVENTINJ.vector = info->intr_core_state.swintr_vector;
@@ -502,7 +713,6 @@ static int update_irq_entry_state(struct guest_info * info) {
                /* reset swintr state */
                info->intr_core_state.swintr_posted = 0;
                info->intr_core_state.swintr_vector = 0;
-               
                break;
            case V3_VIRTUAL_IRQ:
                guest_ctrl->EVENTINJ.type = SVM_INJECTION_IRQ;
@@ -518,6 +728,28 @@ static int update_irq_entry_state(struct guest_info * info) {
     return 0;
 }
 
+int 
+v3_svm_config_tsc_virtualization(struct guest_info * info) {
+    vmcb_ctrl_t * ctrl_area = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data));
+
+
+    if (info->time_state.flags & VM_TIME_TRAP_RDTSC) {
+       ctrl_area->instrs.RDTSC = 1;
+       ctrl_area->svm_instrs.RDTSCP = 1;
+    } else {
+       ctrl_area->instrs.RDTSC = 0;
+       ctrl_area->svm_instrs.RDTSCP = 0;
+
+       if (info->time_state.flags & VM_TIME_TSC_PASSTHROUGH) {
+               ctrl_area->TSC_OFFSET = 0;
+       } else {
+               ctrl_area->TSC_OFFSET = v3_tsc_host_offset(&info->time_state);
+       }
+    }
+    return 0;
+}
+
+
 
 /* 
  * CAUTION and DANGER!!! 
@@ -531,27 +763,28 @@ int v3_svm_enter(struct guest_info * info) {
     vmcb_ctrl_t * guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data));
     vmcb_saved_state_t * guest_state = GET_VMCB_SAVE_STATE_AREA((vmcb_t*)(info->vmm_data)); 
     addr_t exit_code = 0, exit_info1 = 0, exit_info2 = 0;
-    sint64_t tsc_offset;
+    uint64_t guest_cycles = 0;
+
 
     // Conditionally yield the CPU if the timeslice has expired
-    v3_yield_cond(info);
+    v3_schedule(info);
 
-    // Perform any additional yielding needed for time adjustment
-    v3_adjust_time(info);
+#ifdef V3_CONFIG_MEM_TRACK
+    v3_mem_track_entry(info);
+#endif 
 
-    // Check for timeout - since this calls generic hooks in devices
-    // that may do things like pause the VM, it cannot be with interrupts
-    // disabled.
-    v3_check_timeout(info);
+    // Update timer devices after being in the VM before doing 
+    // IRQ updates, so that any interrupts they raise get seen 
+    // immediately.
 
-    // disable global interrupts for vm state transition
-    v3_clgi();
+    v3_advance_time(info, NULL);
 
-    // Update timer devices after being in the VM, with interupts
-    // disabled, but before doing IRQ updates, so that any interrupts they 
-    //raise get seen immediately.
     v3_update_timers(info);
 
+
+    // disable global interrupts for vm state transition
+    v3_clgi();
+
     // Synchronize the guest state to the VMCB
     guest_state->cr0 = info->ctrl_regs.cr0;
     guest_state->cr2 = info->ctrl_regs.cr2;
@@ -559,10 +792,29 @@ int v3_svm_enter(struct guest_info * info) {
     guest_state->cr4 = info->ctrl_regs.cr4;
     guest_state->dr6 = info->dbg_regs.dr6;
     guest_state->dr7 = info->dbg_regs.dr7;
-    guest_ctrl->guest_ctrl.V_TPR = info->ctrl_regs.cr8 & 0xff;
+
+    // CR8 is now updated by read/writes and it contains the APIC TPR
+    // the V_TPR should be just the class part of that.
+    // This update is here just for completeness.  We currently
+    // are ignoring V_TPR on all injections and doing the priority logivc
+    // in the APIC.
+    // guest_ctrl->guest_ctrl.V_TPR = ((info->ctrl_regs.apic_tpr) >> 4) & 0xf;
+
+    //guest_ctrl->guest_ctrl.V_TPR = info->ctrl_regs.cr8 & 0xff;
+    // 
+    
     guest_state->rflags = info->ctrl_regs.rflags;
+
+    // LMA ,LME, SVE?
+
     guest_state->efer = info->ctrl_regs.efer;
     
+    /* Synchronize MSRs */
+    guest_state->star = info->msrs.star;
+    guest_state->lstar = info->msrs.lstar;
+    guest_state->sfmask = info->msrs.sfmask;
+    guest_state->KernelGsBase = info->msrs.kern_gs_base;
+
     guest_state->cpl = info->cpl;
 
     v3_set_vmcb_segments((vmcb_t*)(info->vmm_data), &(info->segments));
@@ -571,19 +823,26 @@ int v3_svm_enter(struct guest_info * info) {
     guest_state->rip = info->rip;
     guest_state->rsp = info->vm_regs.rsp;
 
+    V3_FP_ENTRY_RESTORE(info);
+
 #ifdef V3_CONFIG_SYMCALL
     if (info->sym_core_state.symcall_state.sym_call_active == 0) {
        update_irq_entry_state(info);
     }
 #else 
+
     update_irq_entry_state(info);
 #endif
 
+#ifdef V3_CONFIG_TM_FUNC
+    v3_tm_check_intr_state(info, guest_ctrl, guest_state);
+#endif
+
 
     /* ** */
 
     /*
-      PrintDebug("SVM Entry to CS=%p  rip=%p...\n", 
+      PrintDebug(info->vm_info, info, "SVM Entry to CS=%p  rip=%p...\n", 
       (void *)(addr_t)info->segments.cs.base, 
       (void *)(addr_t)info->rip);
     */
@@ -591,29 +850,55 @@ int v3_svm_enter(struct guest_info * info) {
 #ifdef V3_CONFIG_SYMCALL
     if (info->sym_core_state.symcall_state.sym_call_active == 1) {
        if (guest_ctrl->guest_ctrl.V_IRQ == 1) {
-           V3_Print("!!! Injecting Interrupt during Sym call !!!\n");
+           V3_Print(info->vm_info, info, "!!! Injecting Interrupt during Sym call !!!\n");
        }
     }
 #endif
 
-    v3_time_enter_vm(info);
-    tsc_offset = v3_tsc_host_offset(&info->time_state);
-    guest_ctrl->TSC_OFFSET = tsc_offset;
+    v3_svm_config_tsc_virtualization(info);
 
+    //V3_Print(info->vm_info, info, "Calling v3_svm_launch\n");
+    {  
+       uint64_t entry_tsc = 0;
+       uint64_t exit_tsc = 0;
+       
+#ifdef V3_CONFIG_PWRSTAT_TELEMETRY
+       v3_pwrstat_telemetry_enter(info);
+#endif
+
+#ifdef V3_CONFIG_PMU_TELEMETRY
+       v3_pmu_telemetry_enter(info);
+#endif
 
-    //V3_Print("Calling v3_svm_launch\n");
 
-    v3_svm_launch((vmcb_t *)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcbs[V3_Get_CPU()]);
+       rdtscll(entry_tsc);
 
-    //V3_Print("SVM Returned: Exit Code: %x, guest_rip=%lx\n", (uint32_t)(guest_ctrl->exit_code), (unsigned long)guest_state->rip);
+       v3_svm_launch((vmcb_t *)V3_PAddr(info->vmm_data), &(info->vm_regs), (vmcb_t *)host_vmcbs[V3_Get_CPU()]);
+
+       rdtscll(exit_tsc);
+
+#ifdef V3_CONFIG_PMU_TELEMETRY
+       v3_pmu_telemetry_exit(info);
+#endif
+
+#ifdef V3_CONFIG_PWRSTAT_TELEMETRY
+       v3_pwrstat_telemetry_exit(info);
+#endif
+
+       guest_cycles = exit_tsc - entry_tsc;
+    }
+
+
+    //V3_Print(info->vm_info, info, "SVM Returned: Exit Code: %x, guest_rip=%lx\n", (uint32_t)(guest_ctrl->exit_code), (unsigned long)guest_state->rip);
 
     v3_last_exit = (uint32_t)(guest_ctrl->exit_code);
 
-    // Immediate exit from VM time bookkeeping
-    v3_time_exit_vm(info);
+    v3_advance_time(info, &guest_cycles);
 
     info->num_exits++;
 
+    V3_FP_EXIT_SAVE(info);
+
     // Save Guest state from VMCB
     info->rip = guest_state->rip;
     info->vm_regs.rsp = guest_state->rsp;
@@ -627,10 +912,20 @@ int v3_svm_enter(struct guest_info * info) {
     info->ctrl_regs.cr4 = guest_state->cr4;
     info->dbg_regs.dr6 = guest_state->dr6;
     info->dbg_regs.dr7 = guest_state->dr7;
-    info->ctrl_regs.cr8 = guest_ctrl->guest_ctrl.V_TPR;
+    //
+    // We do not track this anymore
+    // V_TPR is ignored and we do the logic in the APIC
+    //info->ctrl_regs.cr8 = guest_ctrl->guest_ctrl.V_TPR;
+    //
     info->ctrl_regs.rflags = guest_state->rflags;
     info->ctrl_regs.efer = guest_state->efer;
     
+    /* Synchronize MSRs */
+    info->msrs.star =  guest_state->star;
+    info->msrs.lstar = guest_state->lstar;
+    info->msrs.sfmask = guest_state->sfmask;
+    info->msrs.kern_gs_base = guest_state->KernelGsBase;
+
     v3_get_vmcb_segments((vmcb_t*)(info->vmm_data), &(info->segments));
     info->cpu_mode = v3_get_vm_cpu_mode(info);
     info->mem_mode = v3_get_vm_mem_mode(info);
@@ -651,91 +946,169 @@ int v3_svm_enter(struct guest_info * info) {
 
     // reenable global interrupts after vm exit
     v3_stgi();
+
     // Conditionally yield the CPU if the timeslice has expired
-    v3_yield_cond(info);
+    v3_schedule(info);
+
+    // This update timers is for time-dependent handlers
+    // if we're slaved to host time
+    v3_advance_time(info, NULL);
+    v3_update_timers(info);
+
 
     {
        int ret = v3_handle_svm_exit(info, exit_code, exit_info1, exit_info2);
        
        if (ret != 0) {
-           PrintError("Error in SVM exit handler (ret=%d)\n", ret);
-           PrintError("  last Exit was %d (exit code=0x%llx)\n", v3_last_exit, (uint64_t) exit_code);
+           PrintError(info->vm_info, info, "Error in SVM exit handler (ret=%d)\n", ret);
+           PrintError(info->vm_info, info, "  last Exit was %d (exit code=0x%llx)\n", v3_last_exit, (uint64_t) exit_code);
+
            return -1;
        }
     }
 
 
+    if (info->timeouts.timeout_active) {
+       /* Check to see if any timeouts have expired */
+       v3_handle_timeouts(info, guest_cycles);
+    }
+
+#ifdef V3_CONFIG_MEM_TRACK
+    v3_mem_track_exit(info);
+#endif 
+
+
     return 0;
 }
 
-
 int v3_start_svm_guest(struct guest_info * info) {
+
+    int started=0;
+
     //    vmcb_saved_state_t * guest_state = GET_VMCB_SAVE_STATE_AREA((vmcb_t*)(info->vmm_data));
     //  vmcb_ctrl_t * guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data));
 
-    PrintDebug("Starting SVM core %u (on logical core %u)\n", info->vcpu_id, info->pcpu_id);
+    PrintDebug(info->vm_info, info, "Starting SVM core %u (on logical core %u)\n", info->vcpu_id, info->pcpu_id);
 
-    if (info->vcpu_id == 0) {
-       info->core_run_state = CORE_RUNNING;
-    } else  { 
-       PrintDebug("SVM core %u (on %u): Waiting for core initialization\n", info->vcpu_id, info->pcpu_id);
 
-       while (info->core_run_state == CORE_STOPPED) {
-           
-           if (info->vm_info->run_state == VM_STOPPED) {
-               // The VM was stopped before this core was initialized. 
-               return 0;
-           }
+#ifdef V3_CONFIG_MULTIBOOT
+    if (v3_setup_multiboot_core_for_boot(info)) { 
+       PrintError(info->vm_info, info, "Failed to setup Multiboot core...\n");
+       return -1;
+    }
+#endif
 
-           v3_yield(info);
-           //PrintDebug("SVM core %u: still waiting for INIT\n", info->vcpu_id);
-       }
+#ifdef V3_CONFIG_HVM
+    if (v3_setup_hvm_hrt_core_for_boot(info)) { 
+       PrintError(info->vm_info, info, "Failed to setup HRT core...\n");
+       return -1;
+    } 
+#endif
+    while (1) {
 
-       PrintDebug("SVM core %u(on %u) initialized\n", info->vcpu_id, info->pcpu_id);
+       if (info->core_run_state == CORE_STOPPED) {
 
-       // We'll be paranoid about race conditions here
-       v3_wait_at_barrier(info);
-    } 
+           if (info->vcpu_id == 0) {
+               info->core_run_state = CORE_RUNNING;
+           } else  { 
+               PrintDebug(info->vm_info, info, "SVM core %u (on %u): Waiting for core initialization\n", info->vcpu_id, info->pcpu_id);
 
-    PrintDebug("SVM core %u(on %u): I am starting at CS=0x%x (base=0x%p, limit=0x%x),  RIP=0x%p\n", 
-              info->vcpu_id, info->pcpu_id, 
-              info->segments.cs.selector, (void *)(info->segments.cs.base), 
-              info->segments.cs.limit, (void *)(info->rip));
+               V3_NO_WORK(info);
 
+               // Compiler must not optimize away this read
+               while (*((volatile int *)(&info->core_run_state)) == CORE_STOPPED) {
+                   
+                   if (info->vm_info->run_state == VM_STOPPED) {
+                       // The VM was stopped before this core was initialized. 
+                       return 0;
+                   }
+                   
+                   V3_STILL_NO_WORK(info);
 
+                   //PrintDebug(info->vm_info, info, "SVM core %u: still waiting for INIT\n", info->vcpu_id);
+               }
 
-    PrintDebug("SVM core %u: Launching SVM VM (vmcb=%p) (on cpu %u)\n", 
-              info->vcpu_id, (void *)info->vmm_data, info->pcpu_id);
-    //PrintDebugVMCB((vmcb_t*)(info->vmm_data));
-    
-    v3_start_time(info);
+               V3_HAVE_WORK_AGAIN(info);
+               
+               PrintDebug(info->vm_info, info, "SVM core %u(on %u) initialized\n", info->vcpu_id, info->pcpu_id);
+               
+               // We'll be paranoid about race conditions here
+               v3_wait_at_barrier(info);
+           } 
+       }
 
-    while (1) {
+       if (!started) {
+
+           started=1;
+           
+           PrintDebug(info->vm_info, info, "SVM core %u(on %u): I am starting at CS=0x%x (base=0x%p, limit=0x%x),  RIP=0x%p\n", 
+                      info->vcpu_id, info->pcpu_id, 
+                      info->segments.cs.selector, (void *)(info->segments.cs.base), 
+                      info->segments.cs.limit, (void *)(info->rip));
+           
+           
+           
+           PrintDebug(info->vm_info, info, "SVM core %u: Launching SVM VM (vmcb=%p) (on cpu %u)\n", 
+                      info->vcpu_id, (void *)info->vmm_data, info->pcpu_id);
 
+#ifdef V3_CONFIG_DEBUG_SVM
+           PrintDebugVMCB((vmcb_t*)(info->vmm_data));
+#endif
+           
+           v3_start_time(info);
+       }
+       
        if (info->vm_info->run_state == VM_STOPPED) {
            info->core_run_state = CORE_STOPPED;
            break;
        }
        
-       if (v3_svm_enter(info) == -1) {
+       
+#ifdef V3_CONFIG_HVM
+       if (v3_handle_hvm_reset(info) > 0) { 
+           continue;
+       }
+#endif
+       
+#ifdef V3_CONFIG_MULTIBOOT
+       if (v3_handle_multiboot_reset(info) > 0) {
+           continue;
+       }
+#endif
+       
+       if (svm_handle_standard_reset(info) > 0 ) {
+           continue;
+       }
+       
+
+
+#ifdef V3_CONFIG_PMU_TELEMETRY
+       v3_pmu_telemetry_start(info);
+#endif
+       
+#ifdef V3_CONFIG_PWRSTAT_TELEMETRY
+       v3_pwrstat_telemetry_start(info);
+#endif
+       
+       if (v3_svm_enter(info) == -1 ) {
            vmcb_ctrl_t * guest_ctrl = GET_VMCB_CTRL_AREA((vmcb_t*)(info->vmm_data));
            addr_t host_addr;
            addr_t linear_addr = 0;
            
            info->vm_info->run_state = VM_ERROR;
            
-           V3_Print("SVM core %u: SVM ERROR!!\n", info->vcpu_id); 
+           V3_Print(info->vm_info, info, "SVM core %u: SVM ERROR!!\n", info->vcpu_id); 
            
            v3_print_guest_state(info);
            
-           V3_Print("SVM core %u: SVM Exit Code: %p\n", info->vcpu_id, (void *)(addr_t)guest_ctrl->exit_code); 
+           V3_Print(info->vm_info, info, "SVM core %u: SVM Exit Code: %p\n", info->vcpu_id, (void *)(addr_t)guest_ctrl->exit_code); 
            
-           V3_Print("SVM core %u: exit_info1 low = 0x%.8x\n", info->vcpu_id, *(uint_t*)&(guest_ctrl->exit_info1));
-           V3_Print("SVM core %u: exit_info1 high = 0x%.8x\n", info->vcpu_id, *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info1)) + 4));
+           V3_Print(info->vm_info, info, "SVM core %u: exit_info1 low = 0x%.8x\n", info->vcpu_id, *(uint_t*)&(guest_ctrl->exit_info1));
+           V3_Print(info->vm_info, info, "SVM core %u: exit_info1 high = 0x%.8x\n", info->vcpu_id, *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info1)) + 4));
            
-           V3_Print("SVM core %u: exit_info2 low = 0x%.8x\n", info->vcpu_id, *(uint_t*)&(guest_ctrl->exit_info2));
-           V3_Print("SVM core %u: exit_info2 high = 0x%.8x\n", info->vcpu_id, *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info2)) + 4));
+           V3_Print(info->vm_info, info, "SVM core %u: exit_info2 low = 0x%.8x\n", info->vcpu_id, *(uint_t*)&(guest_ctrl->exit_info2));
+           V3_Print(info->vm_info, info, "SVM core %u: exit_info2 high = 0x%.8x\n", info->vcpu_id, *(uint_t *)(((uchar_t *)&(guest_ctrl->exit_info2)) + 4));
            
            linear_addr = get_addr_linear(info, info->rip, &(info->segments.cs));
            
@@ -745,18 +1118,18 @@ int v3_start_svm_guest(struct guest_info * info) {
                v3_gva_to_hva(info, linear_addr, &host_addr);
            }
            
-           V3_Print("SVM core %u: Host Address of rip = 0x%p\n", info->vcpu_id, (void *)host_addr);
+           V3_Print(info->vm_info, info, "SVM core %u: Host Address of rip = 0x%p\n", info->vcpu_id, (void *)host_addr);
            
-           V3_Print("SVM core %u: Instr (15 bytes) at %p:\n", info->vcpu_id, (void *)host_addr);
+           V3_Print(info->vm_info, info, "SVM core %u: Instr (15 bytes) at %p:\n", info->vcpu_id, (void *)host_addr);
            v3_dump_mem((uint8_t *)host_addr, 15);
            
            v3_print_stack(info);
-
+           
            break;
        }
-
+       
        v3_wait_at_barrier(info);
-
+       
 
        if (info->vm_info->run_state == VM_STOPPED) {
            info->core_run_state = CORE_STOPPED;
@@ -767,13 +1140,20 @@ int v3_start_svm_guest(struct guest_info * info) {
 
 /*
        if ((info->num_exits % 50000) == 0) {
-           V3_Print("SVM Exit number %d\n", (uint32_t)info->num_exits);
+           V3_Print(info->vm_info, info, "SVM Exit number %d\n", (uint32_t)info->num_exits);
            v3_print_guest_state(info);
        }
 */
        
     }
 
+#ifdef V3_CONFIG_PMU_TELEMETRY
+    v3_pmu_telemetry_end(info);
+#endif
+
+#ifdef V3_CONFIG_PWRSTAT_TELEMETRY
+    v3_pwrstat_telemetry_end(info);
+#endif
     // Need to take down the other cores on error... 
 
     return 0;
@@ -816,38 +1196,38 @@ int v3_is_svm_capable() {
 
     v3_cpuid(CPUID_EXT_FEATURE_IDS, &eax, &ebx, &ecx, &edx);
   
-    PrintDebug("CPUID_EXT_FEATURE_IDS_ecx=0x%x\n", ecx);
+    PrintDebug(VM_NONE, VCORE_NONE,  "CPUID_EXT_FEATURE_IDS_ecx=0x%x\n", ecx);
 
     if ((ecx & CPUID_EXT_FEATURE_IDS_ecx_svm_avail) == 0) {
-      V3_Print("SVM Not Available\n");
+      V3_Print(VM_NONE, VCORE_NONE,  "SVM Not Available\n");
       return 0;
     }  else {
        v3_get_msr(SVM_VM_CR_MSR, &vm_cr_high, &vm_cr_low);
        
-       PrintDebug("SVM_VM_CR_MSR = 0x%x 0x%x\n", vm_cr_high, vm_cr_low);
+       PrintDebug(VM_NONE, VCORE_NONE, "SVM_VM_CR_MSR = 0x%x 0x%x\n", vm_cr_high, vm_cr_low);
        
        if ((vm_cr_low & SVM_VM_CR_MSR_svmdis) == 1) {
-           V3_Print("SVM is available but is disabled.\n");
+           V3_Print(VM_NONE, VCORE_NONE, "SVM is available but is disabled.\n");
            
            v3_cpuid(CPUID_SVM_REV_AND_FEATURE_IDS, &eax, &ebx, &ecx, &edx);
            
-           PrintDebug("CPUID_SVM_REV_AND_FEATURE_IDS_edx=0x%x\n", edx);
+           PrintDebug(VM_NONE, VCORE_NONE,  "CPUID_SVM_REV_AND_FEATURE_IDS_edx=0x%x\n", edx);
            
            if ((edx & CPUID_SVM_REV_AND_FEATURE_IDS_edx_svml) == 0) {
-               V3_Print("SVM BIOS Disabled, not unlockable\n");
+               V3_Print(VM_NONE, VCORE_NONE,  "SVM BIOS Disabled, not unlockable\n");
            } else {
-               V3_Print("SVM is locked with a key\n");
+               V3_Print(VM_NONE, VCORE_NONE,  "SVM is locked with a key\n");
            }
            return 0;
 
        } else {
-           V3_Print("SVM is available and  enabled.\n");
+           V3_Print(VM_NONE, VCORE_NONE,  "SVM is available and  enabled.\n");
 
            v3_cpuid(CPUID_SVM_REV_AND_FEATURE_IDS, &eax, &ebx, &ecx, &edx);
-           PrintDebug("CPUID_SVM_REV_AND_FEATURE_IDS_eax=0x%x\n", eax);
-           PrintDebug("CPUID_SVM_REV_AND_FEATURE_IDS_ebx=0x%x\n", ebx);
-           PrintDebug("CPUID_SVM_REV_AND_FEATURE_IDS_ecx=0x%x\n", ecx);
-           PrintDebug("CPUID_SVM_REV_AND_FEATURE_IDS_edx=0x%x\n", edx);
+           PrintDebug(VM_NONE, VCORE_NONE, "CPUID_SVM_REV_AND_FEATURE_IDS_eax=0x%x\n", eax);
+           PrintDebug(VM_NONE, VCORE_NONE, "CPUID_SVM_REV_AND_FEATURE_IDS_ebx=0x%x\n", ebx);
+           PrintDebug(VM_NONE, VCORE_NONE, "CPUID_SVM_REV_AND_FEATURE_IDS_ecx=0x%x\n", ecx);
+           PrintDebug(VM_NONE, VCORE_NONE, "CPUID_SVM_REV_AND_FEATURE_IDS_edx=0x%x\n", edx);
 
            return 1;
        }
@@ -859,13 +1239,13 @@ static int has_svm_nested_paging() {
     
     v3_cpuid(CPUID_SVM_REV_AND_FEATURE_IDS, &eax, &ebx, &ecx, &edx);
     
-    //PrintDebug("CPUID_EXT_FEATURE_IDS_edx=0x%x\n", edx);
+    //PrintDebug(VM_NONE, VCORE_NONE,  "CPUID_EXT_FEATURE_IDS_edx=0x%x\n", edx);
     
     if ((edx & CPUID_SVM_REV_AND_FEATURE_IDS_edx_np) == 0) {
-       V3_Print("SVM Nested Paging not supported\n");
+       V3_Print(VM_NONE, VCORE_NONE, "SVM Nested Paging not supported\n");
        return 0;
     } else {
-       V3_Print("SVM Nested Paging supported\n");
+       V3_Print(VM_NONE, VCORE_NONE, "SVM Nested Paging supported\n");
        return 1;
     }
  }
@@ -881,17 +1261,22 @@ void v3_init_svm_cpu(int cpu_id) {
     msr.e_reg.low |= EFER_MSR_svm_enable;
     v3_set_msr(EFER_MSR, 0, msr.e_reg.low);
 
-    V3_Print("SVM Enabled\n");
+    V3_Print(VM_NONE, VCORE_NONE,  "SVM Enabled\n");
 
     // Setup the host state save area
-    host_vmcbs[cpu_id] = (addr_t)V3_AllocPages(4);
+    host_vmcbs[cpu_id] = (addr_t)V3_AllocPages(4); // need not be shadow-safe, not exposed to guest
+
+    if (!host_vmcbs[cpu_id]) {
+       PrintError(VM_NONE, VCORE_NONE,  "Failed to allocate VMCB\n");
+       return;
+    }
 
     /* 64-BIT-ISSUE */
     //  msr.e_reg.high = 0;
     //msr.e_reg.low = (uint_t)host_vmcb;
     msr.r_reg = host_vmcbs[cpu_id];
 
-    PrintDebug("Host State being saved at %p\n", (void *)host_vmcbs[cpu_id]);
+    PrintDebug(VM_NONE, VCORE_NONE,  "Host State being saved at %p\n", (void *)host_vmcbs[cpu_id]);
     v3_set_msr(SVM_VM_HSAVE_PA_MSR, msr.e_reg.high, msr.e_reg.low);
 
 
@@ -922,7 +1307,7 @@ void v3_deinit_svm_cpu(int cpu_id) {
 
     V3_FreePages((void *)host_vmcbs[cpu_id], 4);
 
-    V3_Print("Host CPU %d host area freed, and SVM disabled\n", cpu_id);
+    V3_Print(VM_NONE, VCORE_NONE,  "Host CPU %d host area freed, and SVM disabled\n", cpu_id);
     return;
 }
 
@@ -1005,7 +1390,7 @@ void v3_deinit_svm_cpu(int cpu_id) {
     end <<= 32;
     end += end_lo;
     
-    PrintDebug("VMSave Cycle Latency: %d\n", (uint32_t)(end - start));
+    PrintDebug(core->vm_info, core, "VMSave Cycle Latency: %d\n", (uint32_t)(end - start));
     
     __asm__ __volatile__ (
                          "rdtsc ; "
@@ -1027,7 +1412,7 @@ void v3_deinit_svm_cpu(int cpu_id) {
        end += end_lo;
 
 
-       PrintDebug("VMLoad Cycle Latency: %d\n", (uint32_t)(end - start));
+       PrintDebug(core->vm_info, core, "VMLoad Cycle Latency: %d\n", (uint32_t)(end - start));
     }
     /* End Latency Test */
 
@@ -1039,170 +1424,3 @@ void v3_deinit_svm_cpu(int cpu_id) {
 
 
 
-#if 0
-void Init_VMCB_pe(vmcb_t *vmcb, struct guest_info vm_info) {
-  vmcb_ctrl_t * ctrl_area = GET_VMCB_CTRL_AREA(vmcb);
-  vmcb_saved_state_t * guest_state = GET_VMCB_SAVE_STATE_AREA(vmcb);
-  uint_t i = 0;
-
-
-  guest_state->rsp = vm_info.vm_regs.rsp;
-  guest_state->rip = vm_info.rip;
-
-
-  /* I pretty much just gutted this from TVMM */
-  /* Note: That means its probably wrong */
-
-  // set the segment registers to mirror ours
-  guest_state->cs.selector = 1<<3;
-  guest_state->cs.attrib.fields.type = 0xa; // Code segment+read
-  guest_state->cs.attrib.fields.S = 1;
-  guest_state->cs.attrib.fields.P = 1;
-  guest_state->cs.attrib.fields.db = 1;
-  guest_state->cs.attrib.fields.G = 1;
-  guest_state->cs.limit = 0xfffff;
-  guest_state->cs.base = 0;
-  
-  struct vmcb_selector *segregs [] = {&(guest_state->ss), &(guest_state->ds), &(guest_state->es), &(guest_state->fs), &(guest_state->gs), NULL};
-  for ( i = 0; segregs[i] != NULL; i++) {
-    struct vmcb_selector * seg = segregs[i];
-    
-    seg->selector = 2<<3;
-    seg->attrib.fields.type = 0x2; // Data Segment+read/write
-    seg->attrib.fields.S = 1;
-    seg->attrib.fields.P = 1;
-    seg->attrib.fields.db = 1;
-    seg->attrib.fields.G = 1;
-    seg->limit = 0xfffff;
-    seg->base = 0;
-  }
-
-
-  {
-    /* JRL THIS HAS TO GO */
-    
-    //    guest_state->tr.selector = GetTR_Selector();
-    guest_state->tr.attrib.fields.type = 0x9; 
-    guest_state->tr.attrib.fields.P = 1;
-    // guest_state->tr.limit = GetTR_Limit();
-    //guest_state->tr.base = GetTR_Base();// - 0x2000;
-    /* ** */
-  }
-
-
-  /* ** */
-
-
-  guest_state->efer |= EFER_MSR_svm_enable;
-  guest_state->rflags = 0x00000002; // The reserved bit is always 1
-  ctrl_area->svm_instrs.VMRUN = 1;
-  guest_state->cr0 = 0x00000001;    // PE 
-  ctrl_area->guest_ASID = 1;
-
-
-  //  guest_state->cpl = 0;
-
-
-
-  // Setup exits
-
-  ctrl_area->cr_writes.cr4 = 1;
-  
-  ctrl_area->exceptions.de = 1;
-  ctrl_area->exceptions.df = 1;
-  ctrl_area->exceptions.pf = 1;
-  ctrl_area->exceptions.ts = 1;
-  ctrl_area->exceptions.ss = 1;
-  ctrl_area->exceptions.ac = 1;
-  ctrl_area->exceptions.mc = 1;
-  ctrl_area->exceptions.gp = 1;
-  ctrl_area->exceptions.ud = 1;
-  ctrl_area->exceptions.np = 1;
-  ctrl_area->exceptions.of = 1;
-  ctrl_area->exceptions.nmi = 1;
-
-  
-
-  ctrl_area->instrs.IOIO_PROT = 1;
-  ctrl_area->IOPM_BASE_PA = (uint_t)V3_AllocPages(3);
-  
-  {
-    reg_ex_t tmp_reg;
-    tmp_reg.r_reg = ctrl_area->IOPM_BASE_PA;
-    memset((void*)(tmp_reg.e_reg.low), 0xffffffff, PAGE_SIZE * 2);
-  }
-
-  ctrl_area->instrs.INTR = 1;
-
-  
-  {
-    char gdt_buf[6];
-    char idt_buf[6];
-
-    memset(gdt_buf, 0, 6);
-    memset(idt_buf, 0, 6);
-
-
-    uint_t gdt_base, idt_base;
-    ushort_t gdt_limit, idt_limit;
-    
-    GetGDTR(gdt_buf);
-    gdt_base = *(ulong_t*)((uchar_t*)gdt_buf + 2) & 0xffffffff;
-    gdt_limit = *(ushort_t*)(gdt_buf) & 0xffff;
-    PrintDebug("GDT: base: %x, limit: %x\n", gdt_base, gdt_limit);
-
-    GetIDTR(idt_buf);
-    idt_base = *(ulong_t*)(idt_buf + 2) & 0xffffffff;
-    idt_limit = *(ushort_t*)(idt_buf) & 0xffff;
-    PrintDebug("IDT: base: %x, limit: %x\n",idt_base, idt_limit);
-
-
-    // gdt_base -= 0x2000;
-    //idt_base -= 0x2000;
-
-    guest_state->gdtr.base = gdt_base;
-    guest_state->gdtr.limit = gdt_limit;
-    guest_state->idtr.base = idt_base;
-    guest_state->idtr.limit = idt_limit;
-
-
-  }
-  
-  
-  // also determine if CPU supports nested paging
-  /*
-  if (vm_info.page_tables) {
-    //   if (0) {
-    // Flush the TLB on entries/exits
-    ctrl_area->TLB_CONTROL = 1;
-
-    // Enable Nested Paging
-    ctrl_area->NP_ENABLE = 1;
-
-    PrintDebug("NP_Enable at 0x%x\n", &(ctrl_area->NP_ENABLE));
-
-        // Set the Nested Page Table pointer
-    ctrl_area->N_CR3 |= ((addr_t)vm_info.page_tables & 0xfffff000);
-
-
-    //   ctrl_area->N_CR3 = Get_CR3();
-    // guest_state->cr3 |= (Get_CR3() & 0xfffff000);
-
-    guest_state->g_pat = 0x7040600070406ULL;
-
-    PrintDebug("Set Nested CR3: lo: 0x%x  hi: 0x%x\n", (uint_t)*(&(ctrl_area->N_CR3)), (uint_t)*((unsigned char *)&(ctrl_area->N_CR3) + 4));
-    PrintDebug("Set Guest CR3: lo: 0x%x  hi: 0x%x\n", (uint_t)*(&(guest_state->cr3)), (uint_t)*((unsigned char *)&(guest_state->cr3) + 4));
-    // Enable Paging
-    //    guest_state->cr0 |= 0x80000000;
-  }
-  */
-
-}
-
-
-
-
-
-#endif
-
-