Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


Minor debugging print changes/fixups
[palacios.git] / palacios / src / devices / apic.c
index 2ec7a41..b8e2fef 100644 (file)
 
 #include <devices/apic.h>
 #include <devices/apic_regs.h>
-#include <devices/icc_bus.h>
 #include <palacios/vmm.h>
 #include <palacios/vmm_msr.h>
 #include <palacios/vmm_sprintf.h>
 #include <palacios/vm_guest.h>
+#include <palacios/vmm_types.h>
 
 
 #ifndef CONFIG_DEBUG_APIC
 #define PrintDebug(fmt, args...)
 #endif
 
+#ifdef CONFIG_DEBUG_APIC
+static char *shorthand_str[] = { 
+    "(no shorthand)",
+    "(self)",
+    "(all)",
+    "(all-but-me)",
+};
+
+static char *deliverymode_str[] = { 
+    "(fixed)",
+    "(lowest priority)",
+    "(SMI)",
+    "(reserved)",
+    "(NMI)",
+    "(INIT)",
+    "(Start Up)",
+    "(ExtInt)",
+};
+#endif
 
 typedef enum { APIC_TMR_INT, APIC_THERM_INT, APIC_PERF_INT, 
               APIC_LINT0_INT, APIC_LINT1_INT, APIC_ERR_INT } apic_irq_type_t;
@@ -116,8 +135,6 @@ typedef enum { APIC_TMR_INT, APIC_THERM_INT, APIC_PERF_INT,
 #define EXT_INT_LOC_VEC_TBL_OFFSET2       0x520   // 0x500 - 0x530
 #define EXT_INT_LOC_VEC_TBL_OFFSET3       0x530   // 0x500 - 0x530
 
-
-
 struct apic_msr {
     union {
        uint64_t value;
@@ -126,14 +143,19 @@ struct apic_msr {
            uint8_t bootstrap_cpu : 1;
            uint8_t rsvd2         : 2;
            uint8_t apic_enable   : 1;
-           uint64_t base_addr   : 40;
-           uint32_t rsvd3         : 12;
+           uint64_t base_addr    : 40;
+           uint32_t rsvd3        : 12;
        } __attribute__((packed));
     } __attribute__((packed));
 } __attribute__((packed));
 
 
 
+typedef enum {INIT_ST, 
+             SIPI, 
+             STARTED} ipi_state_t; 
+
+struct apic_dev_state;
 
 struct apic_state {
     addr_t base_addr;
@@ -176,22 +198,35 @@ struct apic_state {
     uint32_t rem_rd_data;
 
 
+    ipi_state_t ipi_state;
+
     uint8_t int_req_reg[32];
     uint8_t int_svc_reg[32];
     uint8_t int_en_reg[32];
     uint8_t trig_mode_reg[32];
-  
-    uint32_t eoi;
 
-    struct vm_device * icc_bus;
+    struct guest_info * core;
+
+    uint32_t eoi;
 
     v3_lock_t  lock;
 };
 
+
+
+
+struct apic_dev_state {
+    int num_apics;
+
+    struct apic_state apics[0];
+} __attribute__((packed));
+
+
+
 static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, uint_t length, void * priv_data);
 static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, uint_t length, void * priv_data);
 
-static void init_apic_state(struct apic_state * apic, uint32_t id, struct vm_device * icc) {
+static void init_apic_state(struct apic_state * apic, uint32_t id) {
     apic->base_addr = DEFAULT_BASE_ADDR;
 
     if (id == 0) { 
@@ -222,7 +257,7 @@ static void init_apic_state(struct apic_state * apic, uint32_t id, struct vm_dev
 
     apic->lapic_id.val = id;
     
-    apic->icc_bus = icc;
+    apic->ipi_state = INIT_ST;
 
     // The P6 has 6 LVT entries, so we set the value to (6-1)...
     apic->apic_ver.val = 0x80050010;
@@ -254,9 +289,8 @@ static void init_apic_state(struct apic_state * apic, uint32_t id, struct vm_dev
 
 
 static int read_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t * dst, void * priv_data) {
-    struct vm_device * dev = (struct vm_device *)priv_data;
-    struct apic_state * apics = (struct apic_state *)(dev->private_data);
-    struct apic_state * apic = &(apics[core->cpu_id]);
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)priv_data;
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]);
 
     PrintDebug("apic %u: core %u: MSR read\n", apic->lapic_id.val, core->cpu_id);
     v3_lock(apic->lock);
@@ -267,13 +301,12 @@ static int read_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t * dst, v
 
 
 static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, void * priv_data) {
-    struct vm_device * dev = (struct vm_device *)priv_data;
-    struct apic_state * apics = (struct apic_state *)(dev->private_data);
-    struct apic_state * apic = &(apics[core->cpu_id]);
-    struct v3_mem_region * old_reg = v3_get_mem_region(dev->vm, core->cpu_id, apic->base_addr);
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)priv_data;
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]);
+    struct v3_mem_region * old_reg = v3_get_mem_region(core->vm_info, core->cpu_id, apic->base_addr);
 
 
-    PrintDebug("apic %u: core %u: MSR write\n",apic->lapic_id.val,core->cpu_id);
+    PrintDebug("apic %u: core %u: MSR write\n", apic->lapic_id.val, core->cpu_id);
 
     if (old_reg == NULL) {
        // uh oh...
@@ -284,11 +317,13 @@ static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, vo
     
     v3_lock(apic->lock);
 
-    v3_delete_mem_region(dev->vm, old_reg);
+    v3_delete_mem_region(core->vm_info, old_reg);
 
     apic->base_addr = src.value;
 
-    if (v3_hook_full_mem(dev->vm, core->cpu_id, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, dev) == -1) {
+    if (v3_hook_full_mem(core->vm_info, core->cpu_id, apic->base_addr, 
+                        apic->base_addr + PAGE_SIZE_4KB, 
+                        apic_read, apic_write, apic_dev) == -1) {
        PrintError("apic %u: core %u: Could not hook new APIC Base address\n",
                   apic->lapic_id.val, core->cpu_id);
        v3_unlock(apic->lock);
@@ -311,22 +346,23 @@ static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
 
 
     if (irq_num <= 15) {
-//     PrintError("apic %u: core ?: Attempting to raise an invalid interrupt: %d\n", apic->lapic_id.val,irq_num);
+       PrintError("apic %u: core %d: Attempting to raise an invalid interrupt: %d\n", 
+                  apic->lapic_id.val, apic->core->cpu_id, irq_num);
        return -1;
     }
 
 
-    PrintDebug("apic %u: core ?: Raising APIC IRQ %d\n", apic->lapic_id.val, irq_num);
+    PrintDebug("apic %u: core %d: Raising APIC IRQ %d\n", apic->lapic_id.val, apic->core->cpu_id, irq_num);
 
     if (*req_location & flag) {
-       //V3_Print("Interrupts coallescing\n");
+       PrintDebug("Interrupt %d  coallescing\n", irq_num);
     }
 
     if (*en_location & flag) {
        *req_location |= flag;
     } else {
-       PrintDebug("apic %u: core ?: Interrupt  not enabled... %.2x\n", 
-                  apic->lapic_id.val, *en_location);
+       PrintDebug("apic %u: core %d: Interrupt  not enabled... %.2x\n", 
+                  apic->lapic_id.val, apic->core->cpu_id,*en_location);
        return 0;
     }
 
@@ -470,8 +506,293 @@ static int activate_internal_irq(struct apic_state * apic, apic_irq_type_t int_t
 }
 
 
+
+static inline int should_deliver_cluster_ipi(struct guest_info * dst_core, 
+                                            struct apic_state * dst_apic, uint8_t mda) {
+
+    if         ( ((mda & 0xf0) == (dst_apic->log_dst.dst_log_id & 0xf0)) &&     // (I am in the cluster and
+         ((mda & 0x0f) & (dst_apic->log_dst.dst_log_id & 0x0f)) ) {  //  I am in the set)
+
+       PrintDebug("apic %u core %u: accepting clustered IRQ (mda 0x%x == log_dst 0x%x)\n",
+                  dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
+                  dst_apic->log_dst.dst_log_id);
+       
+       return 1;
+    } else {
+       PrintDebug("apic %u core %u: rejecting clustered IRQ (mda 0x%x != log_dst 0x%x)\n",
+                  dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
+                  dst_apic->log_dst.dst_log_id);
+       return 0;
+    }
+}
+
+static inline int should_deliver_flat_ipi(struct guest_info * dst_core,
+                                         struct apic_state * dst_apic, uint8_t mda) {
+
+    if (dst_apic->log_dst.dst_log_id & mda) {  // I am in the set 
+
+       PrintDebug("apic %u core %u: accepting flat IRQ (mda 0x%x == log_dst 0x%x)\n",
+                  dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
+                  dst_apic->log_dst.dst_log_id);
+      return 1;
+  } else {
+       PrintDebug("apic %u core %u: rejecting flat IRQ (mda 0x%x != log_dst 0x%x)\n",
+                  dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
+                  dst_apic->log_dst.dst_log_id);
+      return 0;
+  }
+}
+
+
+
+static int should_deliver_ipi(struct guest_info * dst_core, 
+                             struct apic_state * dst_apic, uint8_t mda) {
+
+
+    if (dst_apic->dst_fmt.model == 0xf) {
+
+       if (mda == 0xff) {
+           // always deliver broadcast
+           return 1;
+       }
+
+       return should_deliver_flat_ipi(dst_core, dst_apic, mda);
+    } else if (dst_apic->dst_fmt.model == 0x0) {
+
+       if (mda == 0xff) {
+           // always deliver broadcast
+           return 1;
+       }
+
+       return should_deliver_cluster_ipi(dst_core, dst_apic, mda);
+    } else {
+       PrintError("apic %u core %u: invalid destination format register value 0x%x for logical mode delivery.\n", 
+                  dst_apic->lapic_id.val, dst_core->cpu_id, dst_apic->dst_fmt.model);
+       return -1;
+    }
+}
+
+
+static int deliver_ipi(struct apic_state * src_apic, 
+                      struct apic_state * dst_apic, 
+                      uint32_t vector, uint8_t del_mode) {
+
+    struct guest_info * dst_core = dst_apic->core;
+
+    switch (del_mode) {
+
+       case 0:  //fixed
+       case 1: // lowest priority
+           PrintDebug("delivering IRQ %d to core %u\n", vector, dst_core->cpu_id); 
+
+           activate_apic_irq(dst_apic, vector);
+
+           if (dst_apic != src_apic) { 
+               // Assume core # is same as logical processor for now
+               // TODO FIX THIS FIX THIS
+               // THERE SHOULD BE:  guestapicid->virtualapicid map,
+               //                   cpu_id->logical processor map
+               //     host maitains logical proc->phsysical proc
+               PrintDebug(" non-local core, forcing it to exit\n"); 
+
+               v3_interrupt_cpu(dst_core->vm_info, dst_core->cpu_id, 0);
+           }
+
+           break;
+       case 5: { //INIT
+
+           PrintDebug(" INIT delivery to core %u\n", dst_core->cpu_id);
+
+           // TODO: any APIC reset on dest core (shouldn't be needed, but not sure...)
+
+           // Sanity check
+           if (dst_apic->ipi_state != INIT_ST) { 
+               PrintError(" Warning: core %u is not in INIT state (mode = %d), ignored\n",
+                          dst_core->cpu_id, dst_apic->ipi_state);
+               // Only a warning, since INIT INIT SIPI is common
+               break;
+           }
+
+           // We transition the target core to SIPI state
+           dst_apic->ipi_state = SIPI;  // note: locking should not be needed here
+
+           // That should be it since the target core should be
+           // waiting in host on this transition
+           // either it's on another core or on a different preemptive thread
+           // in both cases, it will quickly notice this transition 
+           // in particular, we should not need to force an exit here
+
+           PrintDebug(" INIT delivery done\n");
+
+           break;                                                      
+       }
+       case 6: { //SIPI
+
+           // Sanity check
+           if (dst_apic->ipi_state != SIPI) { 
+               PrintError(" core %u is not in SIPI state (mode = %d), ignored!\n",
+                          dst_core->cpu_id, dst_apic->ipi_state);
+               break;
+           }
+
+           // Write the RIP, CS, and descriptor
+           // assume the rest is already good to go
+           //
+           // vector VV -> rip at 0
+           //              CS = VV00
+           //  This means we start executing at linear address VV000
+           //
+           // So the selector needs to be VV00
+           // and the base needs to be VV000
+           //
+           dst_core->rip = 0;
+           dst_core->segments.cs.selector = vector << 8;
+           dst_core->segments.cs.limit = 0xffff;
+           dst_core->segments.cs.base = vector << 12;
+
+           PrintDebug(" SIPI delivery (0x%x -> 0x%x:0x0) to core %u\n",
+                      vector, dst_core->segments.cs.selector, dst_core->cpu_id);
+           // Maybe need to adjust the APIC?
+           
+           // We transition the target core to SIPI state
+           dst_core->core_run_state = CORE_RUNNING;  // note: locking should not be needed here
+           dst_apic->ipi_state = STARTED;
+
+           // As with INIT, we should not need to do anything else
+
+           PrintDebug(" SIPI delivery done\n");
+
+           break;                                                      
+       }
+       case 2: // SMI                  
+       case 3: // reserved                                             
+       case 4: // NMI                                  
+       case 7: // ExtInt
+       default:
+           PrintError("IPI %d delivery is unsupported\n", del_mode); 
+           return -1;
+    }
+
+    return 0;
+
+}
+
+
+static int route_ipi(struct apic_dev_state * apic_dev,
+                    struct apic_state * src_apic, 
+                    struct int_cmd_reg * icr) {
+    struct apic_state * dest_apic = NULL;
+
+    PrintDebug("route_ipi: src_apic=%p, icr_data=%p\n", 
+              src_apic, (void *)(addr_t)icr->val);
+
+
+    if ((icr->dst_mode == 0) && (icr->dst >= apic_dev->num_apics)) { 
+       PrintError("route_ipi: Attempted send to unregistered apic id=%u\n", 
+                  icr->dst);
+       return -1;
+    }
+
+    dest_apic =  &(apic_dev->apics[icr->dst]);
+
+
+    PrintDebug("route_ipi: IPI %s %u from apic %p to %s %s %u (icr=0x%llx)\n",
+              deliverymode_str[icr->del_mode], 
+              icr->vec, 
+              src_apic,               
+              (icr->dst_mode == 0) ? "(physical)" : "(logical)", 
+              shorthand_str[icr->dst_shorthand], 
+              icr->dst,
+              icr->val);
+
+    switch (icr->dst_shorthand) {
+
+       case 0:  // no shorthand
+           if (icr->dst_mode == 0) { 
+               // physical delivery
+
+               if (deliver_ipi(src_apic, dest_apic, 
+                               icr->vec, icr->del_mode) == -1) {
+                   PrintError("Error: Could not deliver IPI\n");
+                   return -1;
+               }
+
+           } else {
+               // logical delivery
+               int i;
+               uint8_t mda = icr->dst;
+
+               for (i = 0; i < apic_dev->num_apics; i++) { 
+                    dest_apic = &(apic_dev->apics[i]);
+                    int del_flag = should_deliver_ipi(dest_apic->core, dest_apic, mda);
+                    
+                    if (del_flag == -1) {
+                        PrintError("Error checking delivery mode\n");
+                        return -1;
+                    } else if (del_flag == 1) {
+                       if (deliver_ipi(src_apic, dest_apic, 
+                                       icr->vec, icr->del_mode) == -1) {
+                           PrintError("Error: Could not deliver IPI\n");
+                           return -1;
+                       }
+                   }
+               }
+           }
+           
+           break;
+           
+       case 1:  // self
+
+           if (src_apic == NULL) {
+               PrintError("Sending IPI to self from generic IPI sender\n");
+               break;
+           }
+
+           if (icr->dst_mode == 0) { 
+               if (deliver_ipi(src_apic, src_apic, icr->vec, icr->del_mode) == -1) {
+                   PrintError("Could not deliver IPI\n");
+                   return -1;
+               }
+           } else {
+               // logical delivery
+               PrintError("use of logical delivery in self is not yet supported.\n");
+               return -1;
+           }
+           break;
+           
+       case 2: 
+       case 3: { // all and all-but-me
+           // assuming that logical verus physical doesn't matter
+           // although it is odd that both are used
+           int i;
+
+           for (i = 0; i < apic_dev->num_apics; i++) { 
+               dest_apic = &(apic_dev->apics[i]);
+
+               if ((dest_apic != src_apic) || (icr->dst_shorthand == 2)) { 
+                   if (deliver_ipi(src_apic, dest_apic, icr->vec, icr->del_mode) == -1) {
+                       PrintError("Error: Could not deliver IPI\n");
+                       return -1;
+                   }
+               }
+           }   
+
+           break;
+       }
+       default:
+           PrintError("Error routing IPI, invalid Mode (%d)\n", icr->dst_shorthand);
+           return -1;
+    }
+    
+
+    return 0;
+}
+
+
+
 static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, uint_t length, void * priv_data) {
-    struct apic_state * apic = (struct apic_state *)(priv_data);
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(priv_data);
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]);
     addr_t reg_addr  = guest_addr - apic->base_addr;
     struct apic_msr * msr = (struct apic_msr *)&(apic->base_addr_msr.value);
     uint32_t val = 0;
@@ -734,7 +1055,8 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
  *
  */
 static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, uint_t length, void * priv_data) {
-    struct apic_state * apic = (struct apic_state *)(priv_data);
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(priv_data);
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
     addr_t reg_addr  = guest_addr - apic->base_addr;
     struct apic_msr * msr = (struct apic_msr *)&(apic->base_addr_msr.value);
     uint32_t op_val = *(uint32_t *)src;
@@ -742,8 +1064,8 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
     PrintDebug("apic %u: core %u: at %p and priv_data is at %p\n",
               apic->lapic_id.val, core->cpu_id, apic, priv_data);
 
-    PrintDebug("Write to address space (%p) (val=%x)\n", 
-              (void *)guest_addr, *(uint32_t *)src);
+    PrintDebug("apic %u: core %u: write to address space (%p) (val=%x)\n", 
+              apic->lapic_id.val, core->cpu_id, (void *)guest_addr, *(uint32_t *)src);
 
     if (msr->apic_enable == 0) {
        PrintError("apic %u: core %u: Write to APIC address space with disabled APIC, apic msr=0x%llx\n",
@@ -806,6 +1128,8 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
            apic->task_prio.val = op_val;
            break;
        case LDR_OFFSET:
+           PrintDebug("apic %u: core %u: setting log_dst.val to 0x%x\n",
+                      apic->lapic_id.val, core->cpu_id, op_val);
            apic->log_dst.val = op_val;
            break;
        case DFR_OFFSET:
@@ -896,12 +1220,15 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
        case INT_CMD_LO_OFFSET:
            apic->int_cmd.lo = op_val;
 
-           // ICC???
            PrintDebug("apic %u: core %u: sending cmd 0x%llx to apic %u\n", 
-                      apic->clapic_id.val, core->cpu_id,
+                      apic->lapic_id.val, core->cpu_id,
                       apic->int_cmd.val, apic->int_cmd.dst);
 
-           v3_icc_send_ipi(apic->icc_bus, apic->lapic_id.val, apic->int_cmd.val, 0);
+           if (route_ipi(apic_dev, apic, &(apic->int_cmd)) == -1) { 
+               PrintError("IPI Routing failure\n");
+               return -1;
+           }
+
            break;
 
        case INT_CMD_HI_OFFSET:
@@ -929,8 +1256,9 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
 /* Interrupt Controller Functions */
 
 // returns 1 if an interrupt is pending, 0 otherwise
-static int apic_intr_pending(struct guest_info * info, void * private_data) {
-    struct apic_state * apic = (struct apic_state *)private_data;
+static int apic_intr_pending(struct guest_info * core, void * private_data) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(private_data);
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
     int req_irq = get_highest_irr(apic);
     int svc_irq = get_highest_isr(apic);
 
@@ -944,8 +1272,9 @@ static int apic_intr_pending(struct guest_info * info, void * private_data) {
     return 0;
 }
 
-static int apic_get_intr_number(struct guest_info * info, void * private_data) {
-    struct apic_state * apic = (struct apic_state *)private_data;
+static int apic_get_intr_number(struct guest_info * core, void * private_data) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(private_data);
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
     int req_irq = get_highest_irr(apic);
     int svc_irq = get_highest_isr(apic);
 
@@ -959,16 +1288,48 @@ static int apic_get_intr_number(struct guest_info * info, void * private_data) {
 }
 
 
-static int apic_raise_intr(struct guest_info * info, int irq, void * private_data) {
-  struct apic_state * apic = (struct apic_state *)private_data;
+int v3_apic_send_ipi(struct v3_vm_info * vm, struct vm_device * dev, 
+                    struct v3_gen_ipi * ipi) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(dev->private_data);
+    struct int_cmd_reg tmp_icr;
+
+    // zero out all the fields
+    tmp_icr.val = 0;
+
 
-  return activate_apic_irq(apic, irq);
+    tmp_icr.vec = ipi->vector;
+    tmp_icr.del_mode = ipi->mode;
+    tmp_icr.dst_mode = ipi->logical;
+    tmp_icr.trig_mode = ipi->trigger_mode;
+    tmp_icr.dst_shorthand = ipi->dst_shorthand;
+    tmp_icr.dst = ipi->dst;
+    
+
+    return route_ipi(apic_dev, NULL, &tmp_icr);
 }
 
 
+int v3_apic_raise_intr(struct v3_vm_info * vm, struct vm_device * dev, 
+                      uint32_t irq, uint32_t dst) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(dev->private_data);
+    struct apic_state * apic = &(apic_dev->apics[dst]); 
+
+    PrintDebug("apic %u core ?: raising interrupt IRQ %u (dst = %u).\n", apic->lapic_id.val, irq, dst); 
+
+    activate_apic_irq(apic, irq);
 
-static int apic_begin_irq(struct guest_info * info, void * private_data, int irq) {
-    struct apic_state * apic = (struct apic_state *)private_data;
+    if (V3_Get_CPU() != dst) {
+       v3_interrupt_cpu(vm, dst, 0);
+    }
+
+    return 0;
+}
+
+
+
+static int apic_begin_irq(struct guest_info * core, void * private_data, int irq) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(private_data);
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
     int major_offset = (irq & ~0x00000007) >> 3;
     int minor_offset = irq & 0x00000007;
     uint8_t * req_location = apic->int_req_reg + major_offset;
@@ -982,8 +1343,8 @@ static int apic_begin_irq(struct guest_info * info, void * private_data, int irq
        *req_location &= ~flag;
     } else {
        // do nothing... 
-       PrintDebug("apic %u: core %u: begin irq for %d ignored since I don't own it\n",
-                  apic->lapic_id.val, info->cpu_id, irq);
+       //PrintDebug("apic %u: core %u: begin irq for %d ignored since I don't own it\n",
+       //         apic->lapic_id.val, core->cpu_id, irq);
     }
 
     return 0;
@@ -993,10 +1354,12 @@ static int apic_begin_irq(struct guest_info * info, void * private_data, int irq
 
 
 /* Timer Functions */
-static void apic_update_time(struct guest_info * info, 
+static void apic_update_time(struct guest_info * core, 
                             uint64_t cpu_cycles, uint64_t cpu_freq, 
                             void * priv_data) {
-    struct apic_state * apic = (struct apic_state *)(priv_data);
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(priv_data);
+    struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
+
     // The 32 bit GCC runtime is a pile of shit
 #ifdef __V3_64BIT__
     uint64_t tmr_ticks = 0;
@@ -1046,7 +1409,7 @@ static void apic_update_time(struct guest_info * info,
            break;
        default:
            PrintError("apic %u: core %u: Invalid Timer Divider configuration\n",
-                      apic->lapic_id.val, info->cpu_id);
+                      apic->lapic_id.val, core->cpu_id);
            return;
     }
 
@@ -1061,18 +1424,18 @@ static void apic_update_time(struct guest_info * info,
 
        // raise irq
        PrintDebug("apic %u: core %u: Raising APIC Timer interrupt (periodic=%d) (icnt=%d) (div=%d)\n",
-                  apic->lapic_id.val, info->cpu_id,
+                  apic->lapic_id.val, core->cpu_id,
                   apic->tmr_vec_tbl.tmr_mode, apic->tmr_init_cnt, shift_num);
 
-       if (apic_intr_pending(info, priv_data)) {
+       if (apic_intr_pending(core, priv_data)) {
            PrintDebug("apic %u: core %u: Overriding pending IRQ %d\n", 
-                      apic->lapic_id.val, info->cpu_id, 
-                      apic_get_intr_number(info, priv_data));
+                      apic->lapic_id.val, core->cpu_id, 
+                      apic_get_intr_number(core, priv_data));
        }
 
        if (activate_internal_irq(apic, APIC_TMR_INT) == -1) {
            PrintError("apic %u: core %u: Could not raise Timer interrupt\n",
-                      apic->lapic_id.val, info->cpu_id);
+                      apic->lapic_id.val, core->cpu_id);
        }
     
        if (apic->tmr_vec_tbl.tmr_mode == APIC_TMR_PERIODIC) {
@@ -1093,7 +1456,7 @@ static struct intr_ctrl_ops intr_ops = {
 
 
 static struct vm_timer_ops timer_ops = {
-    .update_time = apic_update_time,
+    .update_timer = apic_update_time,
 };
 
 
@@ -1120,30 +1483,21 @@ static struct v3_device_ops dev_ops = {
 
 
 
-static struct v3_icc_ops icc_ops = {
-    .raise_intr = apic_raise_intr,
-};
-
 
 
 static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
-    PrintDebug("apic: creating an APIC for each core\n");
     char * dev_id = v3_cfg_val(cfg, "ID");
-    char * icc_bus_id = v3_cfg_val(cfg, "bus");
-    struct vm_device * icc = v3_find_dev(vm, icc_bus_id);
-    int i;
+    struct apic_dev_state * apic_dev = NULL;
+    int i = 0;
 
-    if (!icc) {
-        PrintError("apic: Cannot find ICC Bus (%s)\n", icc_bus_id);
-        return -1;
-    }
+    PrintDebug("apic: creating an APIC for each core\n");
 
-    // We allocate one apic per core
-    // APICs are accessed via index which correlates with the core's cpu_id 
-    // 0..num_cores-1   at num_cores is the ioapic (one only)
-    struct apic_state * apic = (struct apic_state *)V3_Malloc(sizeof(struct apic_state) * vm->num_cores);
+    apic_dev = (struct apic_dev_state *)V3_Malloc(sizeof(struct apic_dev_state) + 
+                                                 sizeof(struct apic_state) * vm->num_cores);
 
-    struct vm_device * dev = v3_allocate_device(dev_id, &dev_ops, apic);
+    apic_dev->num_apics = vm->num_cores;
+
+    struct vm_device * dev = v3_allocate_device(dev_id, &dev_ops, apic_dev);
 
     if (v3_attach_device(vm, dev) == -1) {
        PrintError("apic: Could not attach device %s\n", dev_id);
@@ -1152,30 +1506,34 @@ static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
 
     
     for (i = 0; i < vm->num_cores; i++) {
+       struct apic_state * apic = &(apic_dev->apics[i]);
        struct guest_info * core = &(vm->cores[i]);
 
-       init_apic_state(&(apic[i]),i,icc);
-
-       v3_register_intr_controller(core, &intr_ops, &(apic[i]));
+       apic->core = core;
 
-       v3_add_timer(core, &timer_ops, &(apic[i]));
+       init_apic_state(apic, i);
 
-       v3_hook_full_mem(vm, core->cpu_id, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, &(apic[i]));
+       v3_register_intr_controller(core, &intr_ops, apic_dev);
 
-       v3_icc_register_apic(core, icc, i, &icc_ops, &(apic[i]));
+       v3_add_timer(core, &timer_ops, apic_dev);
 
-       PrintDebug("apic %u: (setup device): done, my id is %u\n", i, apic[i].lapic_id.val);
+       v3_hook_full_mem(vm, core->cpu_id, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, apic_dev);
 
+       PrintDebug("apic %u: (setup device): done, my id is %u\n", i, apic->lapic_id.val);
     }
 
+#ifdef CONFIG_DEBUG_APIC
     for (i = 0; i < vm->num_cores; i++) {
+       struct apic_state * apic = &(apic_dev->apics[i]);
        PrintDebug("apic: sanity check: apic %u (at %p) has id %u and msr value %llx\n",
-                  i, &(apic[i]), apic[i].lapic_id.val, apic[i].base_addr_msr.value);
+                  i, apic, apic->lapic_id.val, apic->base_addr_msr.value);
     }
+#endif
+
 
-    PrintDebug("apic: priv_data is at %p\n", apic);
+    PrintDebug("apic: priv_data is at %p\n", apic_dev);
 
-    v3_hook_msr(vm, BASE_ADDR_MSR, read_apic_msr, write_apic_msr, dev);
+    v3_hook_msr(vm, BASE_ADDR_MSR, read_apic_msr, write_apic_msr, apic_dev);
 
     return 0;
 }