Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


Support HVM partitioning for APICs
[palacios.git] / palacios / src / devices / apic.c
index f0a9d04..b5d11b5 100644 (file)
@@ -72,14 +72,6 @@ static char * deliverymode_str[] = {
 typedef enum { APIC_TMR_INT, APIC_THERM_INT, APIC_PERF_INT, 
               APIC_LINT0_INT, APIC_LINT1_INT, APIC_ERR_INT } apic_irq_type_t;
 
-#define APIC_FIXED_DELIVERY  0x0
-#define APIC_LOWEST_DELIVERY 0x1
-#define APIC_SMI_DELIVERY    0x2
-#define APIC_RES1_DELIVERY   0x3
-#define APIC_NMI_DELIVERY    0x4
-#define APIC_INIT_DELIVERY   0x5
-#define APIC_SIPI_DELIVERY   0x6
-#define APIC_EXTINT_DELIVERY 0x7
 
 #define APIC_SHORTHAND_NONE        0x0
 #define APIC_SHORTHAND_SELF        0x1
@@ -180,6 +172,18 @@ struct apic_msr {
 
 
 
+
+struct irq_queue_entry {
+    uint32_t vector;
+    int (*ack)(struct guest_info * core, uint32_t irq, void * private_data);
+    void * private_data;
+
+    struct list_head list_node;
+};
+
+
+
+
 typedef enum {INIT_ST, 
              SIPI, 
              STARTED} ipi_state_t; 
@@ -211,9 +215,9 @@ struct apic_state {
     struct int_cmd_reg int_cmd;
     struct log_dst_reg log_dst;
     struct dst_fmt_reg dst_fmt;
-    struct arb_prio_reg arb_prio;
-    struct task_prio_reg task_prio;
-    struct proc_prio_reg proc_prio;
+    //struct arb_prio_reg arb_prio;     // computed on the fly
+    //struct task_prio_reg task_prio;   // stored in core.ctrl_regs.apic_tpr
+    //struct proc_prio_reg proc_prio;   // computed on the fly
     struct ext_apic_feature_reg ext_apic_feature;
     struct spec_eoi_reg spec_eoi;
   
@@ -234,6 +238,11 @@ struct apic_state {
     uint8_t int_en_reg[32];
     uint8_t trig_mode_reg[32];
 
+    struct {
+       int (*ack)(struct guest_info * core, uint32_t irq, void * private_data);
+       void * private_data;
+    } irq_ack_cbs[256];
+
     struct guest_info * core;
 
     void * controller_handle;
@@ -241,7 +250,12 @@ struct apic_state {
     struct v3_timer * timer;
 
 
-    struct v3_queue irq_queue;
+    struct {
+       v3_lock_t lock;
+       
+       uint64_t num_entries;
+       struct list_head entries;
+    } irq_queue ;
 
     uint32_t eoi;
 
@@ -261,12 +275,23 @@ struct apic_dev_state {
 
 
 
+static void dump_all_apic_state(struct v3_vm_info *vm, struct apic_dev_state *a);
+static void dump_apic_state(struct guest_info *core, struct apic_state * a) ;
 
 
 static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, uint_t length, void * priv_data);
 static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, uint_t length, void * priv_data);
 
-// No lcoking done
+static void set_apic_tpr(struct apic_state *apic, uint32_t val);
+
+static int is_apic_bsp(struct apic_state * apic) {
+    return ((apic->base_addr_msr.value & 0x0000000000000100LL) != 0);
+}
+
+
+
+
+// No locking done
 static void init_apic_state(struct apic_state * apic, uint32_t id) {
     apic->base_addr = DEFAULT_BASE_ADDR;
 
@@ -281,9 +306,9 @@ static void init_apic_state(struct apic_state * apic, uint32_t id) {
     // same base address regardless of ap or main
     apic->base_addr_msr.value |= ((uint64_t)DEFAULT_BASE_ADDR); 
 
-    PrintDebug("apic %u: (init_apic_state): msr=0x%llx\n",id, apic->base_addr_msr.value);
+    PrintDebug(VM_NONE, VCORE_NONE, "apic %u: (init_apic_state): msr=0x%llx\n",id, apic->base_addr_msr.value);
 
-    PrintDebug("apic %u: (init_apic_state): Sizeof Interrupt Request Register %d, should be 32\n",
+    PrintDebug(VM_NONE, VCORE_NONE, "apic %u: (init_apic_state): Sizeof Interrupt Request Register %d, should be 32\n",
               id, (uint_t)sizeof(apic->int_req_reg));
 
     memset(apic->int_req_reg, 0, sizeof(apic->int_req_reg));
@@ -297,16 +322,20 @@ static void init_apic_state(struct apic_state * apic, uint32_t id) {
     apic->tmr_cur_cnt = 0x00000000;
     apic->missed_ints = 0;
 
-    apic->lapic_id.val = id;
+    // note that it's the *lower* 24 bits that are
+    // reserved, not the upper 24.  
+    apic->lapic_id.val = 0;
+    apic->lapic_id.apic_id = id;
     
     apic->ipi_state = INIT_ST;
 
     // The P6 has 6 LVT entries, so we set the value to (6-1)...
     apic->apic_ver.val = 0x80050010;
 
-    apic->task_prio.val = 0x00000000;
-    apic->arb_prio.val = 0x00000000;
-    apic->proc_prio.val = 0x00000000;
+    set_apic_tpr(apic,0x00000000);
+    // note that arbitration priority and processor priority are derived values
+    // and are computed on the fly
+
     apic->log_dst.val = 0x00000000;
     apic->dst_fmt.val = 0xffffffff;
     apic->spurious_int.val = 0x000000ff;
@@ -325,8 +354,9 @@ static void init_apic_state(struct apic_state * apic, uint32_t id) {
     apic->spec_eoi.val = 0x00000000;
 
 
-    v3_init_queue(&(apic->irq_queue));
-
+    INIT_LIST_HEAD(&(apic->irq_queue.entries));
+    v3_lock_init(&(apic->irq_queue.lock));
+    apic->irq_queue.num_entries = 0;
 
 }
 
@@ -338,10 +368,10 @@ static int read_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t * dst, v
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)priv_data;
     struct apic_state * apic = &(apic_dev->apics[core->vcpu_id]);
 
-    PrintDebug("apic %u: core %u: MSR read\n", apic->lapic_id.val, core->vcpu_id);
-
-    dst->value = apic->base_addr;
-
+    PrintDebug(core->vm_info, core, "apic %u: core %u: MSR read getting %llx\n", apic->lapic_id.val, core->vcpu_id, apic->base_addr_msr.value);
+    dst->value = apic->base_addr_msr.value;
     return 0;
 }
 
@@ -352,11 +382,11 @@ static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, vo
     struct v3_mem_region * old_reg = v3_get_mem_region(core->vm_info, core->vcpu_id, apic->base_addr);
 
 
-    PrintDebug("apic %u: core %u: MSR write\n", apic->lapic_id.val, core->vcpu_id);
+    PrintDebug(core->vm_info, core, "apic %u: core %u: MSR write of %llx\n", apic->lapic_id.val, core->vcpu_id, src.value);
 
     if (old_reg == NULL) {
        // uh oh...
-       PrintError("apic %u: core %u: APIC Base address region does not exit...\n",
+       PrintError(core->vm_info, core, "apic %u: core %u: APIC Base address region does not exit...\n",
                   apic->lapic_id.val, core->vcpu_id);
        return -1;
     }
@@ -365,12 +395,18 @@ static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, vo
 
     v3_delete_mem_region(core->vm_info, old_reg);
 
-    apic->base_addr = src.value;
+    apic->base_addr_msr.value = src.value;
+
+    // unhook from old location
+    v3_unhook_mem(core->vm_info,core->vcpu_id,apic->base_addr);
 
+    apic->base_addr = src.value & ~0xfffULL;
+
+    // hook to new location
     if (v3_hook_full_mem(core->vm_info, core->vcpu_id, apic->base_addr, 
                         apic->base_addr + PAGE_SIZE_4KB, 
                         apic_read, apic_write, apic_dev) == -1) {
-       PrintError("apic %u: core %u: Could not hook new APIC Base address\n",
+       PrintError(core->vm_info, core, "apic %u: core %u: Could not hook new APIC Base address\n",
                   apic->lapic_id.val, core->vcpu_id);
 
        return -1;
@@ -385,7 +421,9 @@ static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, vo
 
 
 // irq_num is the bit offset into a 256 bit buffer...
-static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
+static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num, 
+                            int (*ack)(struct guest_info * core, uint32_t irq, void * private_data), 
+                            void * private_data) {
     int major_offset = (irq_num & ~0x00000007) >> 3;
     int minor_offset = irq_num & 0x00000007;
     uint8_t * req_location = apic->int_req_reg + major_offset;
@@ -393,18 +431,21 @@ static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
     uint8_t flag = 0x1 << minor_offset;
 
 
-    PrintDebug("apic %u: core %d: Raising APIC IRQ %d\n", apic->lapic_id.val, apic->core->vcpu_id, irq_num);
+    PrintDebug(VM_NONE, VCORE_NONE, "apic %u: core %d: Raising APIC IRQ %d\n", apic->lapic_id.val, apic->core->vcpu_id, irq_num);
 
     if (*req_location & flag) {
-       PrintDebug("Interrupt %d  coallescing\n", irq_num);
+       PrintDebug(VM_NONE, VCORE_NONE, "Interrupt %d  coallescing\n", irq_num);
        return 0;
     }
 
     if (*en_location & flag) {
        *req_location |= flag;
+       apic->irq_ack_cbs[irq_num].ack = ack;
+       apic->irq_ack_cbs[irq_num].private_data = private_data;
+
        return 1;
     } else {
-       PrintDebug("apic %u: core %d: Interrupt  not enabled... %.2x\n", 
+       PrintDebug(VM_NONE, VCORE_NONE, "apic %u: core %d: Interrupt  not enabled... %.2x\n", 
                   apic->lapic_id.val, apic->core->vcpu_id, *en_location);
     }
 
@@ -412,35 +453,76 @@ static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
 }
 
 
-static int add_apic_irq_entry(struct apic_state * apic, uint8_t irq_num) {
+
+static int add_apic_irq_entry(struct apic_state * apic, uint32_t irq_num, 
+                             int (*ack)(struct guest_info * core, uint32_t irq, void * private_data),
+                             void * private_data) {
+    unsigned int flags = 0;
+    struct irq_queue_entry * entry = NULL;
 
     if (irq_num <= 15) {
-       PrintError("core %d: Attempting to raise an invalid interrupt: %d\n", 
+       PrintError(VM_NONE, VCORE_NONE, "core %d: Attempting to raise an invalid interrupt: %d\n", 
                    apic->core->vcpu_id, irq_num);
        return -1;
     }
 
-    v3_enqueue(&(apic->irq_queue), (addr_t)irq_num);
+    entry = V3_Malloc(sizeof(struct irq_queue_entry));
+
+    if (entry == NULL) {
+       PrintError(VM_NONE, VCORE_NONE, "Could not allocate irq queue entry\n");
+       return -1;
+    }
+
+    entry->vector = irq_num;
+    entry->ack = ack;
+    entry->private_data = private_data;
+
+    flags = v3_lock_irqsave(apic->irq_queue.lock);
+    
+    list_add_tail(&(entry->list_node), &(apic->irq_queue.entries));
+    apic->irq_queue.num_entries++;
+
+    v3_unlock_irqrestore(apic->irq_queue.lock, flags);
+  
 
     return 0;
 }
 
 static void drain_irq_entries(struct apic_state * apic) {
-    uint32_t irq = 0;
 
-    while ((irq = (uint32_t)v3_dequeue(&(apic->irq_queue))) != 0) {
-       activate_apic_irq(apic, irq);
+    while (1) {
+       unsigned int flags = 0;
+       struct irq_queue_entry * entry = NULL;
+    
+       flags = v3_lock_irqsave(apic->irq_queue.lock);
+       
+       if (!list_empty(&(apic->irq_queue.entries))) {
+           struct list_head * q_entry = apic->irq_queue.entries.next;
+           entry = list_entry(q_entry, struct irq_queue_entry, list_node);
+
+           apic->irq_queue.num_entries--;
+           list_del(q_entry);
+       }
+       
+       v3_unlock_irqrestore(apic->irq_queue.lock, flags);
+
+       if (entry == NULL) {
+           break;
+       }
+
+       activate_apic_irq(apic, entry->vector, entry->ack, entry->private_data);
+
+       V3_Free(entry);
     }
 
 }
 
 
 
-
 static int get_highest_isr(struct apic_state * apic) {
     int i = 0, j = 0;
 
-    // We iterate backwards to find the highest priority
+    // We iterate backwards to find the highest priority in-service request
     for (i = 31; i >= 0; i--) {
        uint8_t  * svc_major = apic->int_svc_reg + i;
     
@@ -462,14 +544,15 @@ static int get_highest_isr(struct apic_state * apic) {
 static int get_highest_irr(struct apic_state * apic) {
     int i = 0, j = 0;
 
-    // We iterate backwards to find the highest priority
+    // We iterate backwards to find the highest priority enabled requested interrupt
     for (i = 31; i >= 0; i--) {
        uint8_t  * req_major = apic->int_req_reg + i;
-    
+       uint8_t  * en_major = apic->int_en_reg + i;
+       
        if ((*req_major) & 0xff) {
            for (j = 7; j >= 0; j--) {
                uint8_t flag = 0x1 << j;
-               if ((*req_major) & flag) {
+               if ((*req_major & *en_major) & flag) {
                    return ((i * 8) + j);
                }
            }
@@ -480,9 +563,91 @@ static int get_highest_irr(struct apic_state * apic) {
 }
  
 
+static uint32_t get_isrv(struct apic_state *apic)
+{
+    int isr = get_highest_isr(apic);
+    
+    if (isr>=0) { 
+       return (uint32_t) isr;
+    } else {
+       return 0;
+    }
+}
+
+static uint32_t get_irrv(struct apic_state *apic)
+{
+    int irr = get_highest_irr(apic);
+    
+    if (irr>=0) { 
+       return (uint32_t) irr;
+    } else {
+       return 0;
+    }
+}
+
+
+static uint32_t get_apic_tpr(struct apic_state *apic)
+{
+    return (uint32_t) (apic->core->ctrl_regs.apic_tpr); // see comment in vmm_ctrl_regs.c for how this works
+
+}
+
+static void set_apic_tpr(struct apic_state *apic, uint32_t val)
+{
+    PrintDebug(VM_NONE, VCORE_NONE, "Set apic_tpr to 0x%x from apic reg path\n",val);
+    apic->core->ctrl_regs.apic_tpr = (uint64_t) val; // see comment in vmm_ctrl_regs.c for how this works
+}
+
+static uint32_t get_apic_ppr(struct apic_state *apic)
+{
+    uint32_t tpr = get_apic_tpr(apic);
+    uint32_t isrv = get_isrv(apic);
+    uint32_t tprlevel, isrlevel;
+    uint32_t ppr;
+
+    tprlevel = (tpr >> 4) & 0xf;
+    isrlevel = (isrv >> 4) & 0xf;
+
+    if (tprlevel>=isrlevel) { 
+       ppr = tpr;  // get class and subclass
+    } else { 
+       ppr = (isrlevel << 4); // get class only
+    }
+    
+    return ppr;
+}
+
+
+
+static uint32_t get_apic_apr(struct apic_state *apic)
+{
+    uint32_t tpr = get_apic_tpr(apic);
+    uint32_t isrv = get_isrv(apic);
+    uint32_t irrv = get_irrv(apic);
+    uint32_t tprlevel, isrlevel, irrlevel;
+
+    tprlevel = (tpr >> 4) & 0xf;
+    isrlevel = (isrv >> 4) & 0xf;
+    irrlevel = (irrv >> 4) & 0xf;
+
+    if (tprlevel >= isrlevel) { 
+       if (tprlevel >= irrlevel) { 
+           return tpr;  // get both class and subclass
+       } else {
+           return irrlevel << 4; // get class only
+       }
+    } else {
+       if (isrlevel >= irrlevel) {
+           return isrlevel << 4; // get class only
+       } else {
+           return irrlevel << 4; // get class only
+       }
+    }
+    
+}
 
 
-static int apic_do_eoi(struct apic_state * apic) {
+static int apic_do_eoi(struct guest_info * core, struct apic_state * apic) {
     int isr_irq = get_highest_isr(apic);
 
     if (isr_irq != -1) {
@@ -491,15 +656,19 @@ static int apic_do_eoi(struct apic_state * apic) {
        uint8_t flag = 0x1 << minor_offset;
        uint8_t * svc_location = apic->int_svc_reg + major_offset;
        
-       PrintDebug("apic %u: core ?: Received APIC EOI for IRQ %d\n", apic->lapic_id.val,isr_irq);
+       PrintDebug(core->vm_info, core, "apic %u: core ?: Received APIC EOI for IRQ %d\n", apic->lapic_id.val,isr_irq);
        
        *svc_location &= ~flag;
 
+       if (apic->irq_ack_cbs[isr_irq].ack) {
+           apic->irq_ack_cbs[isr_irq].ack(core, isr_irq, apic->irq_ack_cbs[isr_irq].private_data);
+       }
+
 #ifdef V3_CONFIG_CRAY_XT
        
        if ((isr_irq == 238) || 
            (isr_irq == 239)) {
-           PrintDebug("apic %u: core ?: Acking IRQ %d\n", apic->lapic_id.val,isr_irq);
+           PrintDebug(core->vm_info, core, "apic %u: core ?: Acking IRQ %d\n", apic->lapic_id.val,isr_irq);
        }
        
        if (isr_irq == 238) {
@@ -507,7 +676,7 @@ static int apic_do_eoi(struct apic_state * apic) {
        }
 #endif
     } else {
-       //PrintError("apic %u: core ?: Spurious EOI...\n",apic->lapic_id.val);
+       //PrintError(core->vm_info, core, "apic %u: core ?: Spurious EOI...\n",apic->lapic_id.val);
     }
        
     return 0;
@@ -523,7 +692,7 @@ static int activate_internal_irq(struct apic_state * apic, apic_irq_type_t int_t
     switch (int_type) {
        case APIC_TMR_INT:
            vec_num = apic->tmr_vec_tbl.vec;
-           del_mode = APIC_FIXED_DELIVERY;
+           del_mode = IPI_FIXED;
            masked = apic->tmr_vec_tbl.mask;
            break;
        case APIC_THERM_INT:
@@ -548,25 +717,25 @@ static int activate_internal_irq(struct apic_state * apic, apic_irq_type_t int_t
            break;
        case APIC_ERR_INT:
            vec_num = apic->err_vec_tbl.vec;
-           del_mode = APIC_FIXED_DELIVERY;
+           del_mode = IPI_FIXED;
            masked = apic->err_vec_tbl.mask;
            break;
        default:
-           PrintError("apic %u: core ?: Invalid APIC interrupt type\n", apic->lapic_id.val);
+           PrintError(VM_NONE, VCORE_NONE, "apic %u: core ?: Invalid APIC interrupt type\n", apic->lapic_id.val);
            return -1;
     }
 
     // interrupt is masked, don't send
     if (masked == 1) {
-       PrintDebug("apic %u: core ?: Inerrupt is masked\n", apic->lapic_id.val);
+       PrintDebug(VM_NONE, VCORE_NONE, "apic %u: core ?: Inerrupt is masked\n", apic->lapic_id.val);
        return 0;
     }
 
-    if (del_mode == APIC_FIXED_DELIVERY) {
-       //PrintDebug("Activating internal APIC IRQ %d\n", vec_num);
-       return add_apic_irq_entry(apic, vec_num);
+    if (del_mode == IPI_FIXED) {
+       //PrintDebug(VM_NONE, VCORE_NONE, "Activating internal APIC IRQ %d\n", vec_num);
+       return add_apic_irq_entry(apic, vec_num, NULL, NULL);
     } else {
-       PrintError("apic %u: core ?: Unhandled Delivery Mode\n", apic->lapic_id.val);
+       PrintError(VM_NONE, VCORE_NONE, "apic %u: core ?: Unhandled Delivery Mode\n", apic->lapic_id.val);
        return -1;
     }
 }
@@ -589,11 +758,11 @@ static inline int should_deliver_cluster_ipi(struct apic_dev_state * apic_dev,
 
 
     if (ret == 1) {
-       PrintDebug("apic %u core %u: accepting clustered IRQ (mda 0x%x == log_dst 0x%x)\n",
+       PrintDebug(VM_NONE, VCORE_NONE, "apic %u core %u: accepting clustered IRQ (mda 0x%x == log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->vcpu_id, mda, 
                   dst_apic->log_dst.dst_log_id);
     } else {
-       PrintDebug("apic %u core %u: rejecting clustered IRQ (mda 0x%x != log_dst 0x%x)\n",
+       PrintDebug(VM_NONE, VCORE_NONE, "apic %u core %u: rejecting clustered IRQ (mda 0x%x != log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->vcpu_id, mda, 
                   dst_apic->log_dst.dst_log_id);
     }
@@ -617,11 +786,11 @@ static inline int should_deliver_flat_ipi(struct apic_dev_state * apic_dev,
 
 
     if (ret == 1) {
-       PrintDebug("apic %u core %u: accepting flat IRQ (mda 0x%x == log_dst 0x%x)\n",
+       PrintDebug(VM_NONE, VCORE_NONE, "apic %u core %u: accepting flat IRQ (mda 0x%x == log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->vcpu_id, mda, 
                   dst_apic->log_dst.dst_log_id);
     } else {
-       PrintDebug("apic %u core %u: rejecting flat IRQ (mda 0x%x != log_dst 0x%x)\n",
+       PrintDebug(VM_NONE, VCORE_NONE, "apic %u core %u: rejecting flat IRQ (mda 0x%x != log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->vcpu_id, mda, 
                   dst_apic->log_dst.dst_log_id);
     }
@@ -665,7 +834,7 @@ static int should_deliver_ipi(struct apic_dev_state * apic_dev,
 
 
     if (ret == -1) {
-       PrintError("apic %u core %u: invalid destination format register value 0x%x for logical mode delivery.\n", 
+       PrintError(VM_NONE, VCORE_NONE, "apic %u core %u: invalid destination format register value 0x%x for logical mode delivery.\n", 
                   dst_apic->lapic_id.val, dst_core->vcpu_id, dst_apic->dst_fmt.model);
     }
 
@@ -675,48 +844,66 @@ static int should_deliver_ipi(struct apic_dev_state * apic_dev,
 
 
 
-// Only the src_apic pointer is used
 static int deliver_ipi(struct apic_state * src_apic, 
                       struct apic_state * dst_apic, 
-                      uint32_t vector, uint8_t del_mode) {
+                      struct v3_gen_ipi * ipi) {
 
 
     struct guest_info * dst_core = dst_apic->core;
 
+#ifdef V3_CONFIG_HVM
+
+    // this is the ultimate place where we discrard IPIs that should
+    // not be going to the HRT.  We should have previously
+    // filtered by priority as well - that is, an HRT apic
+    // is not involved in priority calculation for an IPI originating
+    // from a ROS apic or an ioapic or MSI.   On the other hand
+    // an IPI sent from an HRT apic can go anywhere
+    //
+    if (!v3_hvm_should_deliver_ipi(src_apic ? src_apic->core : 0, 
+                                  dst_apic->core)) {
+       PrintDebug(VM_NONE,VCORE_NONE,  
+                  "apic: HVM skipping delivery of IPI from core %u to core %u\n",
+                  src_apic ? src_apic->core ? src_apic->core->vcpu_id : -1 : -1, 
+                  dst_apic->core->vcpu_id);
+       return 0;
+    }
+#endif
 
-    switch (del_mode) {
+    switch (ipi->mode) {
 
-       case APIC_FIXED_DELIVERY:  
-       case APIC_LOWEST_DELIVERY: {
+       case IPI_FIXED:  
+       case IPI_LOWEST_PRIO: {
            // lowest priority - 
            // caller needs to have decided which apic to deliver to!
 
-           PrintDebug("delivering IRQ %d to core %u\n", vector, dst_core->vcpu_id); 
+           PrintDebug(VM_NONE, VCORE_NONE, "delivering IRQ %d to core %u\n", ipi->vector, dst_core->vcpu_id); 
 
-           add_apic_irq_entry(dst_apic, vector);
+           add_apic_irq_entry(dst_apic, ipi->vector, ipi->ack, ipi->private_data);
            
-#ifdef V3_CONFIG_MULTITHREAD_OS
            if (dst_apic != src_apic) { 
-               PrintDebug(" non-local core with new interrupt, forcing it to exit now\n"); 
+               PrintDebug(VM_NONE, VCORE_NONE, " non-local core with new interrupt, forcing it to exit now\n"); 
                v3_interrupt_cpu(dst_core->vm_info, dst_core->pcpu_id, 0);
            }
-#endif
-
 
            break;
        }
-       case APIC_INIT_DELIVERY: { 
+       case IPI_INIT: { 
+
+           PrintDebug(VM_NONE, VCORE_NONE, " INIT delivery to core %u\n", dst_core->vcpu_id);
 
-           PrintDebug(" INIT delivery to core %u\n", dst_core->vcpu_id);
+           if (is_apic_bsp(dst_apic)) {
+               PrintError(VM_NONE, VCORE_NONE, "Attempted to INIT BSP CPU. Ignoring since I have no idea what the hell to do...\n");
+               break;
+           }
 
-           // TODO: any APIC reset on dest core (shouldn't be needed, but not sure...)
 
-           // Sanity check
            if (dst_apic->ipi_state != INIT_ST) { 
-               PrintError(" Warning: core %u is not in INIT state (mode = %d), ignored (assuming this is the deassert)\n",
-                          dst_core->vcpu_id, dst_apic->ipi_state);
-               // Only a warning, since INIT INIT SIPI is common
-               break;
+               v3_raise_barrier(dst_core->vm_info, src_apic->core);
+               dst_core->core_run_state = CORE_STOPPED;
+               dst_apic->ipi_state = INIT_ST;
+               v3_lower_barrier(dst_core->vm_info);
+
            }
 
            // We transition the target core to SIPI state
@@ -728,23 +915,23 @@ static int deliver_ipi(struct apic_state * src_apic,
            // in both cases, it will quickly notice this transition 
            // in particular, we should not need to force an exit here
 
-           PrintDebug(" INIT delivery done\n");
+           PrintDebug(VM_NONE, VCORE_NONE, " INIT delivery done\n");
 
            break;                                                      
        }
-       case APIC_SIPI_DELIVERY: { 
+       case IPI_SIPI: { 
 
            // Sanity check
            if (dst_apic->ipi_state != SIPI) { 
-               PrintError(" core %u is not in SIPI state (mode = %d), ignored!\n",
+               PrintError(VM_NONE, VCORE_NONE, " core %u is not in SIPI state (mode = %d), ignored!\n",
                           dst_core->vcpu_id, dst_apic->ipi_state);
                break;
            }
 
-           v3_reset_vm_core(dst_core, vector);
+           v3_reset_vm_core(dst_core, ipi->vector);
 
-           PrintDebug(" SIPI delivery (0x%x -> 0x%x:0x0) to core %u\n",
-                      vector, dst_core->segments.cs.selector, dst_core->vcpu_id);
+           PrintDebug(VM_NONE, VCORE_NONE, " SIPI delivery (0x%x -> 0x%x:0x0) to core %u\n",
+                      ipi->vector, dst_core->segments.cs.selector, dst_core->vcpu_id);
            // Maybe need to adjust the APIC?
            
            // We transition the target core to SIPI state
@@ -753,12 +940,12 @@ static int deliver_ipi(struct apic_state * src_apic,
            
            // As with INIT, we should not need to do anything else
            
-           PrintDebug(" SIPI delivery done\n");
+           PrintDebug(VM_NONE, VCORE_NONE, " SIPI delivery done\n");
            
            break;                                                      
        }
 
-       case APIC_EXTINT_DELIVERY: // EXTINT
+       case IPI_EXTINT: // EXTINT
            /* Two possible things to do here: 
             * 1. Ignore the IPI and assume the 8259a (PIC) will handle it
             * 2. Add 32 to the vector and inject it...
@@ -766,11 +953,11 @@ static int deliver_ipi(struct apic_state * src_apic,
             */
            return 0;
 
-       case APIC_SMI_DELIVERY: 
-       case APIC_RES1_DELIVERY: // reserved                                            
-       case APIC_NMI_DELIVERY:
+       case IPI_SMI: 
+       case IPI_RES1: // reserved                                              
+       case IPI_NMI:
        default:
-           PrintError("IPI %d delivery is unsupported\n", del_mode); 
+           PrintError(VM_NONE, VCORE_NONE, "IPI %d delivery is unsupported\n", ipi->mode); 
            return -1;
     }
     
@@ -787,13 +974,13 @@ static struct apic_state * find_physical_apic(struct apic_dev_state * apic_dev,
 
     if ( (dst_idx > 0) && (dst_idx < apic_dev->num_apics) ) { 
        // see if it simply is the core id
-       if (apic_dev->apics[dst_idx].lapic_id.val == dst_idx) { 
+       if (apic_dev->apics[dst_idx].lapic_id.apic_id == dst_idx) { 
             dst_apic = &(apic_dev->apics[dst_idx]);
        }
     }
 
     for (i = 0; i < apic_dev->num_apics; i++) { 
-       if (apic_dev->apics[i].lapic_id.val == dst_idx) { 
+       if (apic_dev->apics[i].lapic_id.apic_id == dst_idx) { 
            dst_apic =  &(apic_dev->apics[i]);
        }
     }
@@ -807,52 +994,51 @@ static struct apic_state * find_physical_apic(struct apic_dev_state * apic_dev,
 
 static int route_ipi(struct apic_dev_state * apic_dev,
                     struct apic_state * src_apic, 
-                    struct int_cmd_reg * icr) {
+                    struct v3_gen_ipi * ipi) {
     struct apic_state * dest_apic = NULL;
 
 
-    PrintDebug("apic: IPI %s %u from apic %p to %s %s %u (icr=0x%llx)\n",
-              deliverymode_str[icr->del_mode], 
-              icr->vec, 
+    PrintDebug(VM_NONE, VCORE_NONE, "apic: IPI %s %u from apic %p to %s %s %u\n",
+              deliverymode_str[ipi->mode], 
+              ipi->vector, 
               src_apic,               
-              (icr->dst_mode == 0) ? "(physical)" : "(logical)", 
-              shorthand_str[icr->dst_shorthand], 
-              icr->dst,
-              icr->val);
+              (ipi->logical == 0) ? "(physical)" : "(logical)", 
+              shorthand_str[ipi->dst_shorthand], 
+              ipi->dst);
 
 
-    switch (icr->dst_shorthand) {
+    switch (ipi->dst_shorthand) {
 
        case APIC_SHORTHAND_NONE:  // no shorthand
-           if (icr->dst_mode == APIC_DEST_PHYSICAL) { 
+           if (ipi->logical == APIC_DEST_PHYSICAL) { 
 
-               dest_apic = find_physical_apic(apic_dev, icr->dst);
+               dest_apic = find_physical_apic(apic_dev, ipi->dst);
                
                if (dest_apic == NULL) { 
-                   PrintError("apic: Attempted send to unregistered apic id=%u\n", icr->dst);
+                   PrintError(VM_NONE, VCORE_NONE, "apic: Attempted send to unregistered apic id=%u\n", ipi->dst);
                    return -1;
                }
 
-               if (deliver_ipi(src_apic, dest_apic, 
-                               icr->vec, icr->del_mode) == -1) {
-                   PrintError("apic: Could not deliver IPI\n");
+               if (deliver_ipi(src_apic, dest_apic, ipi) == -1) {
+                   PrintError(VM_NONE, VCORE_NONE, "apic: Could not deliver IPI\n");
                    return -1;
                }
 
 
-               PrintDebug("apic: done\n");
+               PrintDebug(VM_NONE, VCORE_NONE, "apic: done\n");
 
-           } else if (icr->dst_mode == APIC_DEST_LOGICAL) {
+           } else if (ipi->logical == APIC_DEST_LOGICAL) {
                
-               if (icr->del_mode != APIC_LOWEST_DELIVERY) { 
+               if (ipi->mode != IPI_LOWEST_PRIO) { 
                    int i;
-                   uint8_t mda = icr->dst;
+                   uint8_t mda = ipi->dst;
 
                    // logical, but not lowest priority
                    // we immediately trigger
                    // fixed, smi, reserved, nmi, init, sipi, etc
 
-                   
+                   // HVM is handled here within deliver_ipi 
+
                    for (i = 0; i < apic_dev->num_apics; i++) { 
                        int del_flag = 0;
                        
@@ -862,25 +1048,36 @@ static int route_ipi(struct apic_dev_state * apic_dev,
                        
                        if (del_flag == -1) {
 
-                           PrintError("apic: Error checking delivery mode\n");
+                           PrintError(VM_NONE, VCORE_NONE, "apic: Error checking delivery mode\n");
                            return -1;
                        } else if (del_flag == 1) {
 
-                           if (deliver_ipi(src_apic, dest_apic, 
-                                           icr->vec, icr->del_mode) == -1) {
-                               PrintError("apic: Error: Could not deliver IPI\n");
+                           if (deliver_ipi(src_apic, dest_apic, ipi) == -1) {
+                               PrintError(VM_NONE, VCORE_NONE, "apic: Error: Could not deliver IPI\n");
                                return -1;
                            }
                        }
                    }
                } else {  // APIC_LOWEST_DELIVERY
                    struct apic_state * cur_best_apic = NULL;
-                   uint8_t mda = icr->dst;
+                   uint32_t cur_best_apr;
+                   uint8_t mda = ipi->dst;
                    int i;
+                   uint32_t start_apic = 0;
+                   uint32_t num_apics = apic_dev->num_apics;
+
+#ifdef V3_CONFIG_HVM
+                   // Need to limit lowest priority search to ROS apics
+                   // if this is coming from a ROS apic or ioapic, etc. 
+                   v3_hvm_find_apics_seen_by_core(src_apic ? src_apic->core : 0, 
+                                                  apic_dev->apics[0].core->vm_info,
+                                                  &start_apic,
+                                                  &num_apics);
+#endif
 
                    // logical, lowest priority
 
-                   for (i = 0; i < apic_dev->num_apics; i++) { 
+                   for (i = start_apic; i < num_apics; i++) { 
                        int del_flag = 0;
 
                        dest_apic = &(apic_dev->apics[i]);
@@ -888,7 +1085,7 @@ static int route_ipi(struct apic_dev_state * apic_dev,
                        del_flag = should_deliver_ipi(apic_dev, dest_apic->core, dest_apic, mda);
                        
                        if (del_flag == -1) {
-                           PrintError("apic: Error checking delivery mode\n");
+                           PrintError(VM_NONE, VCORE_NONE, "apic: Error checking delivery mode\n");
 
                            return -1;
                        } else if (del_flag == 1) {
@@ -899,25 +1096,29 @@ static int route_ipi(struct apic_dev_state * apic_dev,
 
                            if (cur_best_apic == 0) {
                                cur_best_apic = dest_apic;  
-                           } else if (dest_apic->task_prio.val < cur_best_apic->task_prio.val) {
-                               cur_best_apic = dest_apic;
+                               cur_best_apr = get_apic_apr(dest_apic) & 0xf0;
+                           } else {
+                               uint32_t dest_apr = get_apic_apr(dest_apic) & 0xf0;
+                               if (dest_apr < cur_best_apr) {
+                                   cur_best_apic = dest_apic;
+                                   cur_best_apr = dest_apr;
+                               }
                            } 
 
                            v3_unlock_irqrestore(apic_dev->state_lock, flags);
 
-                       }                       
+                       }
                    }
 
                    // now we will deliver to the best one if it exists
                    if (!cur_best_apic) { 
-                       PrintDebug("apic: lowest priority deliver, but no destinations!\n");
+                       PrintDebug(VM_NONE, VCORE_NONE, "apic: lowest priority delivery, but no destinations!\n");
                    } else {
-                       if (deliver_ipi(src_apic, cur_best_apic, 
-                                       icr->vec, icr->del_mode) == -1) {
-                           PrintError("apic: Error: Could not deliver IPI\n");
+                       if (deliver_ipi(src_apic, cur_best_apic, ipi) == -1) {
+                           PrintError(VM_NONE, VCORE_NONE, "apic: Error: Could not deliver IPI\n");
                            return -1;
                        }
-                       //V3_Print("apic: logical, lowest priority delivery to apic %u\n",cur_best_apic->lapic_id.val);
+                       //V3_Print(VM_NONE, VCORE_NONE, "apic: logical, lowest priority delivery to apic %u\n",cur_best_apic->lapic_id.val);
                    }
                }
            }
@@ -927,22 +1128,22 @@ static int route_ipi(struct apic_dev_state * apic_dev,
        case APIC_SHORTHAND_SELF:  // self
 
            if (src_apic == NULL) {    /* this is not an apic, but it's trying to send to itself??? */
-               PrintError("apic: Sending IPI to self from generic IPI sender\n");
+               PrintError(VM_NONE, VCORE_NONE, "apic: Sending IPI to self from generic IPI sender\n");
                break;
            }
 
 
 
-           if (icr->dst_mode == APIC_DEST_PHYSICAL)  {  /* physical delivery */
-               if (deliver_ipi(src_apic, src_apic, icr->vec, icr->del_mode) == -1) {
-                   PrintError("apic: Could not deliver IPI to self (physical)\n");
+           if (ipi->logical == APIC_DEST_PHYSICAL)  {  /* physical delivery */
+               if (deliver_ipi(src_apic, src_apic, ipi) == -1) {
+                   PrintError(VM_NONE, VCORE_NONE, "apic: Could not deliver IPI to self (physical)\n");
                    return -1;
                }
-           } else if (icr->dst_mode == APIC_DEST_LOGICAL) {  /* logical delivery */
-               PrintError("apic: use of logical delivery in self (untested)\n");
+           } else if (ipi->logical == APIC_DEST_LOGICAL) {  /* logical delivery */
+               PrintError(VM_NONE, VCORE_NONE, "apic: use of logical delivery in self (untested)\n");
 
-               if (deliver_ipi(src_apic, src_apic, icr->vec, icr->del_mode) == -1) {
-                   PrintError("apic: Could not deliver IPI to self (logical)\n");
+               if (deliver_ipi(src_apic, src_apic, ipi) == -1) {
+                   PrintError(VM_NONE, VCORE_NONE, "apic: Could not deliver IPI to self (logical)\n");
                    return -1;
                }
            }
@@ -954,13 +1155,22 @@ static int route_ipi(struct apic_dev_state * apic_dev,
            /* assuming that logical verus physical doesn't matter
               although it is odd that both are used */
            int i;
+           uint32_t start_apic = 0;
+           uint32_t num_apics = apic_dev->num_apics;
+
+#ifdef V3_CONFIG_HVM
+           v3_hvm_find_apics_seen_by_core(src_apic ? src_apic->core : 0, 
+                                          apic_dev->apics[0].core->vm_info,
+                                          &start_apic,
+                                          &num_apics);
+#endif
 
-           for (i = 0; i < apic_dev->num_apics; i++) { 
+           for (i = start_apic; i < num_apics; i++) { 
                dest_apic = &(apic_dev->apics[i]);
                
-               if ((dest_apic != src_apic) || (icr->dst_shorthand == APIC_SHORTHAND_ALL)) { 
-                   if (deliver_ipi(src_apic, dest_apic, icr->vec, icr->del_mode) == -1) {
-                       PrintError("apic: Error: Could not deliver IPI\n");
+               if ((dest_apic != src_apic) || (ipi->dst_shorthand == APIC_SHORTHAND_ALL)) { 
+                   if (deliver_ipi(src_apic, dest_apic, ipi) == -1) {
+                       PrintError(VM_NONE, VCORE_NONE, "apic: Error: Could not deliver IPI\n");
                        return -1;
                    }
                }
@@ -969,7 +1179,7 @@ static int route_ipi(struct apic_dev_state * apic_dev,
            break;
        }
        default:
-           PrintError("apic: Error routing IPI, invalid Mode (%d)\n", icr->dst_shorthand);
+           PrintError(VM_NONE, VCORE_NONE, "apic: Error routing IPI, invalid Mode (%d)\n", ipi->dst_shorthand);
            return -1;
     }
  
@@ -986,11 +1196,11 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
     uint32_t val = 0;
 
 
-    PrintDebug("apic %u: core %u: at %p: Read apic address space (%p)\n",
+    PrintDebug(core->vm_info, core, "apic %u: core %u: at %p: Read apic address space (%p)\n",
               apic->lapic_id.val, core->vcpu_id, apic, (void *)guest_addr);
 
     if (msr->apic_enable == 0) {
-       PrintError("apic %u: core %u: Read from APIC address space with disabled APIC, apic msr=0x%llx\n",
+       PrintError(core->vm_info, core, "apic %u: core %u: Read from APIC address space with disabled APIC, apic msr=0x%llx\n",
                   apic->lapic_id.val, core->vcpu_id, apic->base_addr_msr.value);
        return -1;
     }
@@ -998,7 +1208,7 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
 
     /* Because "May not be supported" doesn't matter to Linux developers... */
     /*   if (length != 4) { */
-    /*     PrintError("Invalid apic read length (%d)\n", length); */
+    /*     PrintError(core->vm_info, core, "Invalid apic read length (%d)\n", length); */
     /*     return -1; */
     /*   } */
 
@@ -1006,7 +1216,7 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
        case EOI_OFFSET:
            // Well, only an idiot would read from a architectural write only register
            // Oh, Hello Linux.
-           //    PrintError("Attempting to read from write only register\n");
+           //    PrintError(core->vm_info, core, "Attempting to read from write only register\n");
            //    return -1;
            break;
 
@@ -1018,13 +1228,13 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
            val = apic->apic_ver.val;
            break;
        case TPR_OFFSET:
-           val = apic->task_prio.val;
+           val = get_apic_tpr(apic);
            break;
        case APR_OFFSET:
-           val = apic->arb_prio.val;
+           val = get_apic_apr(apic);
            break;
        case PPR_OFFSET:
-           val = apic->proc_prio.val;
+           val = get_apic_ppr(apic);
            break;
        case REMOTE_READ_OFFSET:
            val = apic->rem_rd_data;
@@ -1203,7 +1413,7 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
        case SEOI_OFFSET:
 
        default:
-           PrintError("apic %u: core %u: Read from Unhandled APIC Register: %x (getting zero)\n", 
+           PrintError(core->vm_info, core, "apic %u: core %u: Read from Unhandled APIC Register: %x (getting zero)\n", 
                       apic->lapic_id.val, core->vcpu_id, (uint32_t)reg_addr);
            return -1;
     }
@@ -1226,12 +1436,12 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
        *val_ptr = val;
 
     } else {
-       PrintError("apic %u: core %u: Invalid apic read length (%d)\n", 
+       PrintError(core->vm_info, core, "apic %u: core %u: Invalid apic read length (%d)\n", 
                   apic->lapic_id.val, core->vcpu_id, length);
        return -1;
     }
 
-    PrintDebug("apic %u: core %u: Read finished (val=%x)\n", 
+    PrintDebug(core->vm_info, core, "apic %u: core %u: Read finished (val=%x)\n", 
               apic->lapic_id.val, core->vcpu_id, *(uint32_t *)dst);
 
     return length;
@@ -1249,21 +1459,21 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
     uint32_t op_val = *(uint32_t *)src;
     addr_t flags = 0;
 
-    PrintDebug("apic %u: core %u: at %p and priv_data is at %p\n",
+    PrintDebug(core->vm_info, core, "apic %u: core %u: at %p and priv_data is at %p\n",
               apic->lapic_id.val, core->vcpu_id, apic, priv_data);
 
-    PrintDebug("apic %u: core %u: write to address space (%p) (val=%x)\n", 
+    PrintDebug(core->vm_info, core, "apic %u: core %u: write to address space (%p) (val=%x)\n", 
               apic->lapic_id.val, core->vcpu_id, (void *)guest_addr, *(uint32_t *)src);
 
     if (msr->apic_enable == 0) {
-       PrintError("apic %u: core %u: Write to APIC address space with disabled APIC, apic msr=0x%llx\n",
+       PrintError(core->vm_info, core, "apic %u: core %u: Write to APIC address space with disabled APIC, apic msr=0x%llx\n",
                   apic->lapic_id.val, core->vcpu_id, apic->base_addr_msr.value);
        return -1;
     }
 
 
     if (length != 4) {
-       PrintError("apic %u: core %u: Invalid apic write length (%d)\n", 
+       PrintError(core->vm_info, core, "apic %u: core %u: Invalid apic write length (%d)\n", 
                   apic->lapic_id.val, length, core->vcpu_id);
        return -1;
     }
@@ -1299,23 +1509,23 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
        case PPR_OFFSET:
        case EXT_APIC_FEATURE_OFFSET:
 
-           PrintError("apic %u: core %u: Attempting to write to read only register %p (error)\n", 
+           PrintError(core->vm_info, core, "apic %u: core %u: Attempting to write to read only register %p (error)\n", 
                       apic->lapic_id.val, core->vcpu_id, (void *)reg_addr);
 
            break;
 
            // Data registers
        case APIC_ID_OFFSET:
-           //V3_Print("apic %u: core %u: my id is being changed to %u\n", 
+           //V3_Print(core->vm_info, core, "apic %u: core %u: my id is being changed to %u\n", 
            //       apic->lapic_id.val, core->vcpu_id, op_val);
 
            apic->lapic_id.val = op_val;
            break;
        case TPR_OFFSET:
-           apic->task_prio.val = op_val;
+           set_apic_tpr(apic,op_val);
            break;
        case LDR_OFFSET:
-           PrintDebug("apic %u: core %u: setting log_dst.val to 0x%x\n",
+           PrintDebug(core->vm_info, core, "apic %u: core %u: setting log_dst.val to 0x%x\n",
                       apic->lapic_id.val, core->vcpu_id, op_val);
            flags = v3_lock_irqsave(apic_dev->state_lock);
            apic->log_dst.val = op_val;
@@ -1358,7 +1568,7 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
            apic->tmr_cur_cnt = op_val;
            break;
        case TMR_DIV_CFG_OFFSET:
-           PrintDebug("apic %u: core %u: setting tmr_div_cfg to 0x%x\n",
+           PrintDebug(core->vm_info, core, "apic %u: core %u: setting tmr_div_cfg to 0x%x\n",
                       apic->lapic_id.val, core->vcpu_id, op_val);
            apic->tmr_div_cfg.val = op_val;
            break;
@@ -1407,24 +1617,33 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
            // Action Registers
        case EOI_OFFSET:
            // do eoi 
-           apic_do_eoi(apic);
+           apic_do_eoi(core, apic);
            break;
 
        case INT_CMD_LO_OFFSET: {
            // execute command 
 
-           struct int_cmd_reg tmp_icr;
+           struct v3_gen_ipi tmp_ipi;
 
            apic->int_cmd.lo = op_val;
 
-           tmp_icr = apic->int_cmd;
+           tmp_ipi.vector = apic->int_cmd.vec;
+           tmp_ipi.mode = apic->int_cmd.del_mode;
+           tmp_ipi.logical = apic->int_cmd.dst_mode;
+           tmp_ipi.trigger_mode = apic->int_cmd.trig_mode;
+           tmp_ipi.dst_shorthand = apic->int_cmd.dst_shorthand;
+           tmp_ipi.dst = apic->int_cmd.dst;
+               
+           tmp_ipi.ack = NULL;
+           tmp_ipi.private_data = NULL;
+           
 
-           //      V3_Print("apic %u: core %u: sending cmd 0x%llx to apic %u\n", 
+           //      V3_Print(core->vm_info, core, "apic %u: core %u: sending cmd 0x%llx to apic %u\n", 
            //       apic->lapic_id.val, core->vcpu_id,
            //       apic->int_cmd.val, apic->int_cmd.dst);
 
-           if (route_ipi(apic_dev, apic, &tmp_icr) == -1) { 
-               PrintError("IPI Routing failure\n");
+           if (route_ipi(apic_dev, apic, &tmp_ipi) == -1) { 
+               PrintError(core->vm_info, core, "IPI Routing failure\n");
                return -1;
            }
 
@@ -1432,21 +1651,20 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
        }
        case INT_CMD_HI_OFFSET: {
            apic->int_cmd.hi = op_val;
-           V3_Print("apic %u: core %u: writing command high=0x%x\n", apic->lapic_id.val, core->vcpu_id,apic->int_cmd.hi);
-
+           //V3_Print(core->vm_info, core, "apic %u: core %u: writing command high=0x%x\n", apic->lapic_id.val, core->vcpu_id,apic->int_cmd.hi);
            break;
        }
        // Unhandled Registers
        case EXT_APIC_CMD_OFFSET:
        case SEOI_OFFSET:
        default:
-           PrintError("apic %u: core %u: Write to Unhandled APIC Register: %x (ignored)\n", 
+           PrintError(core->vm_info, core, "apic %u: core %u: Write to Unhandled APIC Register: %x (ignored)\n", 
                       apic->lapic_id.val, core->vcpu_id, (uint32_t)reg_addr);
 
            return -1;
     }
 
-    PrintDebug("apic %u: core %u: Write finished\n", apic->lapic_id.val, core->vcpu_id);
+    PrintDebug(core->vm_info, core, "apic %u: core %u: Write finished\n", apic->lapic_id.val, core->vcpu_id);
 
     return length;
 
@@ -1470,14 +1688,35 @@ static int apic_intr_pending(struct guest_info * core, void * private_data) {
     req_irq = get_highest_irr(apic);
     svc_irq = get_highest_isr(apic);
 
-    //    PrintDebug("apic %u: core %u: req_irq=%d, svc_irq=%d\n",apic->lapic_id.val,info->vcpu_id,req_irq,svc_irq);
+    //    PrintDebug(core->vm_info, core, "apic %u: core %u: req_irq=%d, svc_irq=%d\n",apic->lapic_id.val,info->vcpu_id,req_irq,svc_irq);
+
 
     if ((req_irq >= 0) && 
        (req_irq > svc_irq)) {
-       return 1;
-    }
 
-    return 0;
+       // We have a new requested vector that is higher priority than
+       // the vector that is in-service
+
+       uint32_t ppr = get_apic_ppr(apic);
+
+       if ((req_irq & 0xf0) > (ppr & 0xf0)) {
+           // it's also higher priority than the current
+           // processor priority.  Therefore this
+           // interrupt can go in now.
+           return 1;
+       } else {
+           // processor priority is currently too high
+           // for this interrupt to go in now.  
+           // note that if tpr=0xf?, then ppr=0xf?
+           // and thus all vectors will be masked
+           // as required (tpr=0xf? => all masked)
+           return 0;
+       }
+    } else {
+       // the vector that is in service is higher
+       // priority than any new requested vector
+       return 0;
+    }
 }
 
 
@@ -1488,13 +1727,23 @@ static int apic_get_intr_number(struct guest_info * core, void * private_data) {
     int req_irq = get_highest_irr(apic);
     int svc_irq = get_highest_isr(apic);
 
-    if (svc_irq == -1) {
-       return req_irq;
-    } else if (svc_irq < req_irq) {
-       return req_irq;
-    }
 
-    return -1;
+    // for the logic here, see the comments for apic_intr_pending
+    if ((req_irq >=0) &&
+       (req_irq > svc_irq)) { 
+       
+       uint32_t ppr = get_apic_ppr(apic);
+       
+       if ((req_irq & 0xf0) > (ppr & 0xf0)) {
+           return req_irq;
+       } else {
+           // hmm, this should not have happened, but, anyway,
+           // no interrupt is currently ready to go in
+           return -1;
+       }
+    } else {
+       return -1;
+    }
 }
 
 
@@ -1502,39 +1751,8 @@ static int apic_get_intr_number(struct guest_info * core, void * private_data) {
 int v3_apic_send_ipi(struct v3_vm_info * vm, struct v3_gen_ipi * ipi, void * dev_data) {
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)
        (((struct vm_device *)dev_data)->private_data);
-    struct int_cmd_reg tmp_icr;
-
-    // zero out all the fields
-    tmp_icr.val = 0;
-
-    tmp_icr.vec = ipi->vector;
-    tmp_icr.del_mode = ipi->mode;
-    tmp_icr.dst_mode = ipi->logical;
-    tmp_icr.trig_mode = ipi->trigger_mode;
-    tmp_icr.dst_shorthand = ipi->dst_shorthand;
-    tmp_icr.dst = ipi->dst;
-
-
-    return route_ipi(apic_dev, NULL, &tmp_icr);
-}
-
-
-int v3_apic_raise_intr(struct v3_vm_info * vm, uint32_t irq, uint32_t dst, void * dev_data) {
-    struct apic_dev_state * apic_dev = (struct apic_dev_state *)
-       (((struct vm_device*)dev_data)->private_data);
-    struct apic_state * apic = &(apic_dev->apics[dst]); 
-
-    PrintDebug("apic %u core ?: raising interrupt IRQ %u (dst = %u).\n", apic->lapic_id.val, irq, dst); 
-
-    add_apic_irq_entry(apic, irq);
-
-#ifdef V3_CONFIG_MULTITHREAD_OS   
-    if ((V3_Get_CPU() != dst)) {
-       v3_interrupt_cpu(vm, dst, 0);
-    }
-#endif
 
-    return 0;
+    return route_ipi(apic_dev, NULL, ipi);
 }
 
 
@@ -1555,7 +1773,7 @@ static int apic_begin_irq(struct guest_info * core, void * private_data, int irq
        *req_location &= ~flag;
     } else {
        // do nothing... 
-       //PrintDebug("apic %u: core %u: begin irq for %d ignored since I don't own it\n",
+       //PrintDebug(core->vm_info, core, "apic %u: core %u: begin irq for %d ignored since I don't own it\n",
        //         apic->lapic_id.val, core->vcpu_id, irq);
     }
 
@@ -1570,18 +1788,18 @@ static void apic_inject_timer_intr(struct guest_info *core,
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)(priv_data);
     struct apic_state * apic = &(apic_dev->apics[core->vcpu_id]); 
     // raise irq
-    PrintDebug("apic %u: core %u: Raising APIC Timer interrupt (periodic=%d) (icnt=%d)\n",
+    PrintDebug(core->vm_info, core, "apic %u: core %u: Raising APIC Timer interrupt (periodic=%d) (icnt=%d)\n",
               apic->lapic_id.val, core->vcpu_id,
               apic->tmr_vec_tbl.tmr_mode, apic->tmr_init_cnt);
 
     if (apic_intr_pending(core, priv_data)) {
-        PrintDebug("apic %u: core %u: Overriding pending IRQ %d\n", 
+        PrintDebug(core->vm_info, core, "apic %u: core %u: Overriding pending IRQ %d\n", 
                   apic->lapic_id.val, core->vcpu_id, 
                   apic_get_intr_number(core, priv_data));
     }
 
     if (activate_internal_irq(apic, APIC_TMR_INT) == -1) {
-       PrintError("apic %u: core %u: Could not raise Timer interrupt\n",
+       PrintError(core->vm_info, core, "apic %u: core %u: Could not raise Timer interrupt\n",
                   apic->lapic_id.val, core->vcpu_id);
     }
 
@@ -1614,7 +1832,7 @@ static void apic_update_time(struct guest_info * core,
     if ((apic->tmr_init_cnt == 0) || 
        ( (apic->tmr_vec_tbl.tmr_mode == APIC_TMR_ONESHOT) &&
          (apic->tmr_cur_cnt == 0))) {
-       //PrintDebug("apic %u: core %u: APIC timer not yet initialized\n",apic->lapic_id.val,info->vcpu_id);
+       //PrintDebug(core->vm_info, core, "apic %u: core %u: APIC timer not yet initialized\n",apic->lapic_id.val,info->vcpu_id);
        return;
     }
 
@@ -1645,22 +1863,24 @@ static void apic_update_time(struct guest_info * core,
            shift_num = 7;
            break;
        default:
-           PrintError("apic %u: core %u: Invalid Timer Divider configuration\n",
+           PrintError(core->vm_info, core, "apic %u: core %u: Invalid Timer Divider configuration\n",
                       apic->lapic_id.val, core->vcpu_id);
            return;
     }
 
     tmr_ticks = cpu_cycles >> shift_num;
-    //    PrintDebug("Timer Ticks: %p\n", (void *)tmr_ticks);
+    //    PrintDebug(core->vm_info, core, "Timer Ticks: %p\n", (void *)tmr_ticks);
 
     if (tmr_ticks < apic->tmr_cur_cnt) {
        apic->tmr_cur_cnt -= tmr_ticks;
-       if (apic->missed_ints) {
-           PrintDebug("apic %u: core %u: Injecting queued APIC timer interrupt.\n",
+#ifdef V3_CONFIG_APIC_ENQUEUE_MISSED_TMR_IRQS
+       if (apic->missed_ints && !apic_intr_pending(core, priv_data)) {
+           PrintDebug(core->vm_info, core, "apic %u: core %u: Injecting queued APIC timer interrupt.\n",
                       apic->lapic_id.val, core->vcpu_id);
            apic_inject_timer_intr(core, priv_data);
            apic->missed_ints--;
        }
+#endif /* CONFIG_APIC_ENQUEUE_MISSED_TMR_IRQS */ 
     } else {
        tmr_ticks -= apic->tmr_cur_cnt;
        apic->tmr_cur_cnt = 0;
@@ -1669,12 +1889,9 @@ static void apic_update_time(struct guest_info * core,
 
        if (apic->tmr_vec_tbl.tmr_mode == APIC_TMR_PERIODIC) {
            int queued_ints = tmr_ticks / apic->tmr_init_cnt;
-           if (queued_ints)
-               PrintDebug("apic %u: core %u: Deferring %u APIC timer interrupts.\n",
-                          apic->lapic_id.val, core->vcpu_id, queued_ints);
-           apic->missed_ints += queued_ints;
            tmr_ticks = tmr_ticks % apic->tmr_init_cnt;
            apic->tmr_cur_cnt = apic->tmr_init_cnt - tmr_ticks;
+           apic->missed_ints += queued_ints;
        }
     }
 
@@ -1711,109 +1928,221 @@ static int apic_free(struct apic_dev_state * apic_dev) {
            v3_remove_timer(core, apic->timer);
        }
 
-       // unhook memory
+       v3_lock_deinit(&(apic->irq_queue.lock));
+
+       v3_unhook_mem(vm,core->vcpu_id,apic->base_addr);
 
     }
 
     v3_unhook_msr(vm, BASE_ADDR_MSR);
 
+    v3_lock_deinit(&(apic_dev->state_lock));
+
     V3_Free(apic_dev);
     return 0;
 }
 
 #ifdef V3_CONFIG_CHECKPOINT
+
+#define KEY_MAX 128
+#define MAKE_KEY(x) snprintf(key,KEY_MAX,"%s%d",x,i);
+
 static int apic_save(struct v3_chkpt_ctx * ctx, void * private_data) {
     struct apic_dev_state * apic_state = (struct apic_dev_state *)private_data;
     int i = 0;
+    uint32_t temp;
+    char key[KEY_MAX];
 
-    V3_CHKPT_STD_SAVE(ctx, apic_state->num_apics);
+    V3_CHKPT_SAVE(ctx, "NUM_APICS", apic_state->num_apics,savefailout);
 
-    //V3_CHKPT_STD_SAVE(ctx,apic_state->state_lock);
     for (i = 0; i < apic_state->num_apics; i++) {
-
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].base_addr);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].base_addr_msr);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].lapic_id);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].apic_ver);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ext_apic_ctrl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].local_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_div_cfg);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].lint0_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].lint1_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].perf_ctr_loc_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].therm_loc_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].err_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].err_status);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].spurious_int);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_cmd);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].log_dst);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].dst_fmt);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].arb_prio);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].task_prio);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].proc_prio);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ext_apic_feature);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].spec_eoi);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_cur_cnt);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].tmr_init_cnt);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ext_intr_vec_tbl);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].rem_rd_data);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].ipi_state);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_req_reg);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_svc_reg);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].int_en_reg);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].trig_mode_reg);
-       V3_CHKPT_STD_SAVE(ctx, apic_state->apics[i].eoi);
+      drain_irq_entries(&(apic_state->apics[i]));
+
+      MAKE_KEY("BASE_ADDR"); 
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].base_addr,savefailout);
+      MAKE_KEY("BASE_ADDR_MSR"); 
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].base_addr_msr,savefailout);
+      MAKE_KEY("LAPIC_ID");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].lapic_id,savefailout);
+      MAKE_KEY("APIC_VER");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].apic_ver,savefailout);
+      MAKE_KEY("EXT_APIC_CTRL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].ext_apic_ctrl,savefailout);
+      MAKE_KEY("LOCAL_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].local_vec_tbl,savefailout);
+      MAKE_KEY("TMR_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].tmr_vec_tbl,savefailout);
+      MAKE_KEY("TMR_DIV_CFG");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].tmr_div_cfg,savefailout);
+      MAKE_KEY("LINT0_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].lint0_vec_tbl,savefailout);
+      MAKE_KEY("LINT1_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].lint1_vec_tbl,savefailout);
+      MAKE_KEY("PERF_CTR_LOC_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].perf_ctr_loc_vec_tbl,savefailout);
+      MAKE_KEY("THERM_LOC_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].therm_loc_vec_tbl,savefailout);
+      MAKE_KEY("ERR_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].err_vec_tbl,savefailout);
+      MAKE_KEY("ERR_STATUS");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].err_status,savefailout);
+      MAKE_KEY("SPURIOUS_INT");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].spurious_int,savefailout);
+      MAKE_KEY("INT_CMD");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].int_cmd,savefailout);
+      MAKE_KEY("LOG_DST");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].log_dst,savefailout);
+      MAKE_KEY("DST_FMT");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].dst_fmt,savefailout);
+
+      // APR and PPR are stored only for compatability
+      // TPR is in APIC_TPR, APR and PPR are derived
+       
+      temp = get_apic_apr(&(apic_state->apics[i]));
+      MAKE_KEY("ARB_PRIO");
+      V3_CHKPT_SAVE(ctx, key, temp,savefailout);
+      temp = get_apic_tpr(&(apic_state->apics[i]));
+      MAKE_KEY("TASK_PRIO");
+      V3_CHKPT_SAVE(ctx,key,temp,savefailout);
+      temp = get_apic_ppr(&(apic_state->apics[i]));
+      MAKE_KEY("PROC_PRIO");
+      V3_CHKPT_SAVE(ctx, key,temp,savefailout);
+
+      MAKE_KEY("EXT_APIC_FEATURE");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].ext_apic_feature,savefailout);
+      MAKE_KEY("SPEC_EOI");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].spec_eoi,savefailout);
+      MAKE_KEY("TMR_CUR_CNT");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].tmr_cur_cnt,savefailout);
+      
+      MAKE_KEY("TMR_INIT_CNT");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].tmr_init_cnt,savefailout);
+      MAKE_KEY("EXT_INTR_VEC_TBL");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].ext_intr_vec_tbl,savefailout);
+
+      MAKE_KEY("REM_RD_DATA");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].rem_rd_data,savefailout);
+      MAKE_KEY("IPI_STATE");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].ipi_state,savefailout);
+      MAKE_KEY("INT_REQ_REG");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].int_req_reg,savefailout);
+      MAKE_KEY("INT_SVC_REG");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].int_svc_reg,savefailout);
+      MAKE_KEY("INT_EN_REG");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].int_en_reg,savefailout);
+      MAKE_KEY("TRIG_MODE_REG");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].trig_mode_reg,savefailout);
+      MAKE_KEY("EOI");
+      V3_CHKPT_SAVE(ctx, key, apic_state->apics[i].eoi,savefailout);
 
     }
 
     return 0;
+
+ savefailout:
+    PrintError(VM_NONE, VCORE_NONE, "Failed to save apic\n");
+    return -1;
 }
 
 static int apic_load(struct v3_chkpt_ctx * ctx, void * private_data) {
     struct apic_dev_state *apic_state = (struct apic_dev_state *)private_data;
     int i = 0;
+    uint32_t temp;
+    char key[KEY_MAX];
 
-    V3_CHKPT_STD_LOAD(ctx,apic_state->num_apics);
+    V3_CHKPT_LOAD(ctx,"NUM_APICS", apic_state->num_apics, loadfailout);
 
     for (i = 0; i < apic_state->num_apics; i++) {
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].base_addr);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].base_addr_msr);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].lapic_id);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].apic_ver);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ext_apic_ctrl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].local_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_div_cfg);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].lint0_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].lint1_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].perf_ctr_loc_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].therm_loc_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].err_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].err_status);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].spurious_int);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_cmd);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].log_dst);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].dst_fmt);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].arb_prio);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].task_prio);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].proc_prio);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ext_apic_feature);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].spec_eoi);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_cur_cnt);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].tmr_init_cnt);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ext_intr_vec_tbl);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].rem_rd_data);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].ipi_state);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_req_reg);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_svc_reg);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].int_en_reg);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].trig_mode_reg);
-       V3_CHKPT_STD_LOAD(ctx, apic_state->apics[i].eoi);
+      drain_irq_entries(&(apic_state->apics[i]));
+
+      MAKE_KEY("BASE_ADDR"); 
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].base_addr,loadfailout);
+      MAKE_KEY("BASE_ADDR_MSR"); 
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].base_addr_msr,loadfailout);
+      MAKE_KEY("LAPIC_ID");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].lapic_id,loadfailout);
+      MAKE_KEY("APIC_VER");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].apic_ver,loadfailout);
+      MAKE_KEY("EXT_APIC_CTRL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].ext_apic_ctrl,loadfailout);
+      MAKE_KEY("LOCAL_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].local_vec_tbl,loadfailout);
+      MAKE_KEY("TMR_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].tmr_vec_tbl,loadfailout);
+      MAKE_KEY("TMR_DIV_CFG");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].tmr_div_cfg,loadfailout);
+      MAKE_KEY("LINT0_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].lint0_vec_tbl,loadfailout);
+      MAKE_KEY("LINT1_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].lint1_vec_tbl,loadfailout);
+      MAKE_KEY("PERF_CTR_LOC_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].perf_ctr_loc_vec_tbl,loadfailout);
+      MAKE_KEY("THERM_LOC_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].therm_loc_vec_tbl,loadfailout);
+      MAKE_KEY("ERR_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].err_vec_tbl,loadfailout);
+      MAKE_KEY("ERR_STATUS");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].err_status,loadfailout);
+      MAKE_KEY("SPURIOUS_INT");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].spurious_int,loadfailout);
+      MAKE_KEY("INT_CMD");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].int_cmd,loadfailout);
+      MAKE_KEY("LOG_DST");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].log_dst,loadfailout);
+      MAKE_KEY("DST_FMT");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].dst_fmt,loadfailout);
+
+      // APR and PPR are stored only for compatability
+      // TPR is in APIC_TPR, APR and PPR are derived
+
+      MAKE_KEY("ARB_PRIO");
+      V3_CHKPT_LOAD(ctx, key, temp,loadfailout);
+      // discarded
+
+      MAKE_KEY("TASK_PRIO");
+      V3_CHKPT_LOAD(ctx,key,temp,loadfailout);
+      set_apic_tpr(&(apic_state->apics[i]),temp);
+      
+      MAKE_KEY("PROC_PRIO");
+      V3_CHKPT_LOAD(ctx, key,temp,loadfailout);
+      // discarded
+
+
+      MAKE_KEY("EXT_APIC_FEATURE");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].ext_apic_feature,loadfailout);
+      MAKE_KEY("SPEC_EOI");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].spec_eoi,loadfailout);
+      MAKE_KEY("TMR_CUR_CNT");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].tmr_cur_cnt,loadfailout);
+      
+      MAKE_KEY("TMR_INIT_CNT");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].tmr_init_cnt,loadfailout);
+      MAKE_KEY("EXT_INTR_VEC_TBL");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].ext_intr_vec_tbl,loadfailout);
+
+      MAKE_KEY("REM_RD_DATA");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].rem_rd_data,loadfailout);
+      MAKE_KEY("IPI_STATE");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].ipi_state,loadfailout);
+      MAKE_KEY("INT_REQ_REG");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].int_req_reg,loadfailout);
+      MAKE_KEY("INT_SVC_REG");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].int_svc_reg,loadfailout);
+      MAKE_KEY("INT_EN_REG");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].int_en_reg,loadfailout);
+      MAKE_KEY("TRIG_MODE_REG");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].trig_mode_reg,loadfailout);
+      MAKE_KEY("EOI");
+      V3_CHKPT_LOAD(ctx, key, apic_state->apics[i].eoi,loadfailout);
     }
-
-
+    
+    
     return 0;
+    
+ loadfailout:
+    PrintError(VM_NONE,VCORE_NONE, "Failed to load apic\n");
+    return -1;
+    
 }
 
 #endif
@@ -1833,18 +2162,28 @@ static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
     struct apic_dev_state * apic_dev = NULL;
     int i = 0;
 
-    PrintDebug("apic: creating an APIC for each core\n");
+    PrintDebug(vm, VCORE_NONE, "apic: creating an APIC for each core\n");
 
     apic_dev = (struct apic_dev_state *)V3_Malloc(sizeof(struct apic_dev_state) + 
                                                  sizeof(struct apic_state) * vm->num_cores);
 
+
+    if (!apic_dev) {
+       PrintError(vm, VCORE_NONE, "Failed to allocate space for APIC\n");
+       return -1;
+    }
+
+    memset(apic_dev,0,
+          sizeof(struct apic_dev_state) + 
+          sizeof(struct apic_state) * vm->num_cores);
+
     apic_dev->num_apics = vm->num_cores;
     v3_lock_init(&(apic_dev->state_lock));
 
     struct vm_device * dev = v3_add_device(vm, dev_id, &dev_ops, apic_dev);
 
     if (dev == NULL) {
-       PrintError("apic: Could not attach device %s\n", dev_id);
+       PrintError(vm, VCORE_NONE, "apic: Could not attach device %s\n", dev_id);
        V3_Free(apic_dev);
        return -1;
     }
@@ -1863,32 +2202,310 @@ static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
        apic->timer = v3_add_timer(core, &timer_ops, apic_dev);
 
        if (apic->timer == NULL) {
-           PrintError("APIC: Failed to attach timer to core %d\n", i);
+           PrintError(vm, VCORE_NONE,"APIC: Failed to attach timer to core %d\n", i);
            v3_remove_device(dev);
            return -1;
        }
 
+       // hook to initial location
        v3_hook_full_mem(vm, core->vcpu_id, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, apic_dev);
 
-       PrintDebug("apic %u: (setup device): done, my id is %u\n", i, apic->lapic_id.val);
+       PrintDebug(vm, VCORE_NONE, "apic %u: (setup device): done, my id is %u\n", i, apic->lapic_id.val);
     }
 
 #ifdef V3_CONFIG_DEBUG_APIC
     for (i = 0; i < vm->num_cores; i++) {
        struct apic_state * apic = &(apic_dev->apics[i]);
-       PrintDebug("apic: sanity check: apic %u (at %p) has id %u and msr value %llx and core at %p\n",
+       PrintDebug(vm, VCORE_NONE, "apic: sanity check: apic %u (at %p) has id %u and msr value %llx and core at %p\n",
                   i, apic, apic->lapic_id.val, apic->base_addr_msr.value,apic->core);
     }
 #endif
 
 
-    PrintDebug("apic: priv_data is at %p\n", apic_dev);
+    PrintDebug(vm, VCORE_NONE, "apic: priv_data is at %p\n", apic_dev);
 
     v3_hook_msr(vm, BASE_ADDR_MSR, read_apic_msr, write_apic_msr, apic_dev);
 
     return 0;
 }
 
+static char hexify_nybble(char c)
+{
+    if (c>=0 && c<=9) { 
+       return '0'+c;
+    } else if (c>=0xa && c<=0xf) { 
+       return 'a'+(c-0xa);
+    } else {
+       return -1;
+    }
+}
+
+
+static int hexify_byte(char *c, char b)
+{
+    char n;
+    n = hexify_nybble( (b >> 4) & 0xf);
+    if (n==-1) { 
+       return -1;
+    }
+    c[0] = n;
+    n = hexify_nybble( b & 0xf);
+    if (n==-1) { 
+       return -1;
+    }
+    c[1] = n;
+    return 0;
+}
+
+// dest must be of length at least 2*n+1
+static int hexify_byte_string(char *dest, char *src, int n)
+{
+    int i;
+    for (i=0;i<n;i++) { 
+       if (hexify_byte(dest,src[i])) { 
+           return -1;
+       }
+       dest+=2;
+    }
+    *dest=0;
+    return 0;
+}
+
+
+static __attribute__((unused)) void dump_all_apic_state(struct v3_vm_info *vm, struct apic_dev_state *a)
+{
+    int i;
+    for (i=0;i<a->num_apics;i++) { 
+       dump_apic_state(&(vm->cores[i]),&(a->apics[i]));
+    }
+}
+       
+static void dump_apic_state(struct guest_info *core, struct apic_state * a) 
+{
+    char buf[80];
+    struct irq_queue_entry *ie;
+
+    V3_Print(core->vm_info, core, "APIC (vcore %d) {\n", core->vcpu_id);
+    V3_Print(core->vm_info, core, "\tbase_addr: %llx\n", (uint64_t)(a->base_addr));
+    V3_Print(core->vm_info, core, "\tlapic_id_reg {\n");
+    V3_Print(core->vm_info, core, "\t\trsvd: 0x%x\n", a->lapic_id.rsvd);
+    V3_Print(core->vm_info, core, "\t\tapic_id: 0x%x\n", a->lapic_id.apic_id);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tapic_ver_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tver: 0x%x\n", a->apic_ver.ver);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->apic_ver.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tmax_lvts: 0x%x\n", a->apic_ver.max_lvts);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->apic_ver.rsvd2);
+    V3_Print(core->vm_info, core, "\t\text_reg_present: 0x%x\n", a->apic_ver.ext_reg_present);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\text_apic_ctrl_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tver: 0x%x\n", a->ext_apic_ctrl.ver);
+    V3_Print(core->vm_info, core, "\t\tseoi_enable: 0x%x\n", a->ext_apic_ctrl.seoi_enable);
+    V3_Print(core->vm_info, core, "\t\text_id_enable: 0x%x\n", a->ext_apic_ctrl.ext_id_enable);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->ext_apic_ctrl.rsvd2);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tlocal_vec_tbl_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->local_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->local_vec_tbl.msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->local_vec_tbl.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->local_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->local_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->local_vec_tbl.rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->local_vec_tbl.trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->local_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\ttmr_mode: 0x%x\n", a->local_vec_tbl.tmr_mode);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->local_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\ttmr_vec_tbl_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->tmr_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\trsvd: 0x%x\n", a->tmr_vec_tbl.rsvd);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->tmr_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->tmr_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->tmr_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\ttmr_mode: 0x%x\n", a->tmr_vec_tbl.tmr_mode);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->tmr_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\ttmr_div_cfg_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tdiv_val: 0x%x\n", a->tmr_div_cfg.div_val);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->tmr_div_cfg.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdiv_val2: 0x%x\n", a->tmr_div_cfg.div_val2);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->tmr_div_cfg.rsvd2);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tlint_vec_tbl_reg 0 {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->lint0_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->lint0_vec_tbl.msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->lint0_vec_tbl.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->lint0_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->lint0_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->lint0_vec_tbl.rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->lint0_vec_tbl.trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->lint0_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->lint0_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tlint_vec_tbl_reg 1 {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->lint1_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->lint1_vec_tbl.msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->lint1_vec_tbl.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->lint1_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->lint1_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->lint1_vec_tbl.rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->lint1_vec_tbl.trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->lint1_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->lint1_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tperf_ctr_loc_vec_tbl_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->perf_ctr_loc_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->perf_ctr_loc_vec_tbl.msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->perf_ctr_loc_vec_tbl.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->perf_ctr_loc_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->perf_ctr_loc_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->perf_ctr_loc_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->perf_ctr_loc_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\ttherm_loc_vec_tbl_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->therm_loc_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->therm_loc_vec_tbl.msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->therm_loc_vec_tbl.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->therm_loc_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->therm_loc_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->therm_loc_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->therm_loc_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\terr_vec_tbl_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->err_vec_tbl.vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->err_vec_tbl.msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->err_vec_tbl.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->err_vec_tbl.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->err_vec_tbl.rsvd2);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->err_vec_tbl.mask);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->err_vec_tbl.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\terr_status_reg {\n");
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->err_status.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tsent_acc_err: 0x%x\n", a->err_status.sent_acc_err);
+    V3_Print(core->vm_info, core, "\t\trecv_acc_err: 0x%x\n", a->err_status.recv_acc_err);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->err_status.rsvd2);
+    V3_Print(core->vm_info, core, "\t\tsent_ill_err: 0x%x\n", a->err_status.sent_ill_err);
+    V3_Print(core->vm_info, core, "\t\trecv_ill_err: 0x%x\n", a->err_status.recv_ill_err);
+    V3_Print(core->vm_info, core, "\t\till_reg_addr: 0x%x\n", a->err_status.ill_reg_addr);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->err_status.rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tspurious_int_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->spurious_int.vec);
+    V3_Print(core->vm_info, core, "\t\tapic_soft_en: 0x%x\n", a->spurious_int.apic_soft_en);
+    V3_Print(core->vm_info, core, "\t\tfoc_cpu_chk: 0x%x\n", a->spurious_int.foc_cpu_chk);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->spurious_int.rsvd1);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tint_cmd_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->int_cmd.vec);
+    V3_Print(core->vm_info, core, "\t\tdel_mode: 0x%x\n", a->int_cmd.del_mode);
+    V3_Print(core->vm_info, core, "\t\tdst_mode: 0x%x\n", a->int_cmd.dst_mode);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->int_cmd.del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->int_cmd.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tlvl: 0x%x\n", a->int_cmd.lvl);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->int_cmd.trig_mode);
+    V3_Print(core->vm_info, core, "\t\trm_rd_status: 0x%x\n", a->int_cmd.rem_rd_status);
+    V3_Print(core->vm_info, core, "\t\tdst_shorthand: 0x%x\n", a->int_cmd.dst_shorthand);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%llx\n", (uint64_t)(a->int_cmd.rsvd2));
+    V3_Print(core->vm_info, core, "\t\tdst: 0x%x\n", a->int_cmd.dst);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tlog_dst_reg {\n");
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->log_dst.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdst_log_id: 0x%x\n", a->log_dst.dst_log_id);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tdst_fmt_reg {\n");
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->dst_fmt.rsvd1);
+    V3_Print(core->vm_info, core, "\t\tmodel: 0x%x\n", a->dst_fmt.model);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tarb_prio_reg: 0x%x\n",get_apic_apr(a));
+    V3_Print(core->vm_info, core, "\ttask_prio_reg: 0x%x\n", get_apic_tpr(a));
+    V3_Print(core->vm_info, core, "\tproc_prio_reg: 0x%x\n",get_apic_ppr(a));
+    V3_Print(core->vm_info, core, "\text_apic_feature_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tint_en_reg_cap: 0x%x\n", a->ext_apic_feature.int_en_reg_cap);
+    V3_Print(core->vm_info, core, "\t\tspec_eoi_cap: 0x%x\n", a->ext_apic_feature.spec_eoi_cap);
+    V3_Print(core->vm_info, core, "\t\text_apic_id_cap: 0x%x\n", a->ext_apic_feature.ext_apic_id_cap);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->ext_apic_feature.rsvd1);
+    V3_Print(core->vm_info, core, "\t\text_lvt_cnt: 0x%x\n", a->ext_apic_feature.ext_lvt_cnt);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->ext_apic_feature.rsvd2);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\tspec_eoi_reg {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->spec_eoi.vec);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->spec_eoi.rsvd1);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\ttmr_cur_cnt: 0x%x\n", a->tmr_cur_cnt);
+    V3_Print(core->vm_info, core, "\ttmr_init_cnt: 0x%x\n", a->tmr_init_cnt);
+    V3_Print(core->vm_info, core, "\tmissed_ints: 0x%x\n", a->missed_ints);
+    V3_Print(core->vm_info, core, "\text_vec_tbl_reg 0 {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->ext_intr_vec_tbl[0].vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->ext_intr_vec_tbl[0].msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->ext_intr_vec_tbl[0].rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->ext_intr_vec_tbl[0].del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->ext_intr_vec_tbl[0].rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->ext_intr_vec_tbl[0].rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->ext_intr_vec_tbl[0].trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->ext_intr_vec_tbl[0].mask);
+    V3_Print(core->vm_info, core, "\t\ttmr_mode: 0x%x\n", a->ext_intr_vec_tbl[0].tmr_mode);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->ext_intr_vec_tbl[0].rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\text_vec_tbl_reg 1 {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->ext_intr_vec_tbl[1].vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->ext_intr_vec_tbl[1].msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->ext_intr_vec_tbl[1].rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->ext_intr_vec_tbl[1].del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->ext_intr_vec_tbl[1].rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->ext_intr_vec_tbl[1].rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->ext_intr_vec_tbl[1].trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->ext_intr_vec_tbl[1].mask);
+    V3_Print(core->vm_info, core, "\t\ttmr_mode: 0x%x\n", a->ext_intr_vec_tbl[1].tmr_mode);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->ext_intr_vec_tbl[1].rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\text_vec_tbl_reg 2 {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->ext_intr_vec_tbl[2].vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->ext_intr_vec_tbl[2].msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->ext_intr_vec_tbl[2].rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->ext_intr_vec_tbl[2].del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->ext_intr_vec_tbl[2].rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->ext_intr_vec_tbl[2].rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->ext_intr_vec_tbl[2].trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->ext_intr_vec_tbl[2].mask);
+    V3_Print(core->vm_info, core, "\t\ttmr_mode: 0x%x\n", a->ext_intr_vec_tbl[2].tmr_mode);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->ext_intr_vec_tbl[2].rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\text_vec_tbl_reg 3 {\n");
+    V3_Print(core->vm_info, core, "\t\tvec: 0x%x\n", a->ext_intr_vec_tbl[3].vec);
+    V3_Print(core->vm_info, core, "\t\tmsg_type: 0x%x\n", a->ext_intr_vec_tbl[3].msg_type);
+    V3_Print(core->vm_info, core, "\t\trsvd1: 0x%x\n", a->ext_intr_vec_tbl[3].rsvd1);
+    V3_Print(core->vm_info, core, "\t\tdel_status: 0x%x\n", a->ext_intr_vec_tbl[3].del_status);
+    V3_Print(core->vm_info, core, "\t\trsvd2: 0x%x\n", a->ext_intr_vec_tbl[3].rsvd2);
+    V3_Print(core->vm_info, core, "\t\trem_irr: 0x%x\n", a->ext_intr_vec_tbl[3].rem_irr);
+    V3_Print(core->vm_info, core, "\t\ttrig_mode: 0x%x\n", a->ext_intr_vec_tbl[3].trig_mode);
+    V3_Print(core->vm_info, core, "\t\tmask: 0x%x\n", a->ext_intr_vec_tbl[3].mask);
+    V3_Print(core->vm_info, core, "\t\ttmr_mode: 0x%x\n", a->ext_intr_vec_tbl[3].tmr_mode);
+    V3_Print(core->vm_info, core, "\t\trsvd3: 0x%x\n", a->ext_intr_vec_tbl[3].rsvd3);
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\trem_rd_data: 0x%x\n", a->rem_rd_data);
+    hexify_byte_string(buf,a->int_req_reg,32);
+    V3_Print(core->vm_info, core, "\tint_req_reg: 0x%s\n",buf);
+    hexify_byte_string(buf,a->int_svc_reg,32);
+    V3_Print(core->vm_info, core, "\tint_svc_reg: 0x%s\n",buf);
+    hexify_byte_string(buf,a->int_en_reg,32);
+    V3_Print(core->vm_info, core, "\tint_en_reg: 0x%s\n",buf);
+    hexify_byte_string(buf,a->trig_mode_reg,32);
+    V3_Print(core->vm_info, core, "\ttrig_mode_reg: 0x%s\n",buf);
+    V3_Print(core->vm_info, core, "\tirq_ack_cbs: SKIPPED\n");
+    V3_Print(core->vm_info, core, "\tirq_queue: (follows)\n");
+    // note we do not hold the lock for purposes of printing this list... 
+    list_for_each_entry(ie,&(a->irq_queue.entries), list_node) {
+       V3_Print(core->vm_info,core,"\t\tvector 0x%x ack %p priv %p\n", ie->vector, ie->ack, ie->private_data);
+    }
+    V3_Print(core->vm_info, core, "\t}\n");
+    V3_Print(core->vm_info, core, "\teoi: 0x%x\n", a->eoi);
+    V3_Print(core->vm_info, core,"}\n");
+}
+
+
+
+
 
 
 device_register("LAPIC", apic_init)