Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


build fixes to merge the Palacios configuration parameters with Linux parameters.
[palacios.git] / palacios / src / devices / apic.c
index b8e2fef..7f2eb45 100644 (file)
@@ -11,7 +11,8 @@
  * Copyright (c) 2008, The V3VEE Project <http://www.v3vee.org> 
  * All rights reserved.
  *
- * Author: Jack Lange <jarusl@cs.northwestern.edu>
+ * Authors: Jack Lange <jarusl@cs.northwestern.edu>
+ *          Peter Dinda <pdinda@northwestern.edu> (SMP)
  *
  * This is free software.  You are permitted to use,
  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
 #include <palacios/vmm_types.h>
 
 
-#ifndef CONFIG_DEBUG_APIC
+
+
+#ifndef V3_CONFIG_DEBUG_APIC
 #undef PrintDebug
 #define PrintDebug(fmt, args...)
-#endif
+#else
 
-#ifdef CONFIG_DEBUG_APIC
-static char *shorthand_str[] = { 
+static char * shorthand_str[] = { 
     "(no shorthand)",
     "(self)",
     "(all)",
     "(all-but-me)",
 };
 
-static char *deliverymode_str[] = { 
+static char * deliverymode_str[] = { 
     "(fixed)",
     "(lowest priority)",
     "(SMI)",
@@ -50,17 +52,29 @@ static char *deliverymode_str[] = {
     "(Start Up)",
     "(ExtInt)",
 };
+
 #endif
 
 typedef enum { APIC_TMR_INT, APIC_THERM_INT, APIC_PERF_INT, 
               APIC_LINT0_INT, APIC_LINT1_INT, APIC_ERR_INT } apic_irq_type_t;
 
 #define APIC_FIXED_DELIVERY  0x0
+#define APIC_LOWEST_DELIVERY 0x1
 #define APIC_SMI_DELIVERY    0x2
+#define APIC_RES1_DELIVERY   0x3
 #define APIC_NMI_DELIVERY    0x4
 #define APIC_INIT_DELIVERY   0x5
+#define APIC_SIPI_DELIVERY   0x6
 #define APIC_EXTINT_DELIVERY 0x7
 
+#define APIC_SHORTHAND_NONE        0x0
+#define APIC_SHORTHAND_SELF        0x1
+#define APIC_SHORTHAND_ALL         0x2
+#define APIC_SHORTHAND_ALL_BUT_ME  0x3
+
+#define APIC_DEST_PHYSICAL    0x0
+#define APIC_DEST_LOGICAL     0x1
+
 
 #define BASE_ADDR_MSR     0x0000001B
 #define DEFAULT_BASE_ADDR 0xfee00000
@@ -207,9 +221,13 @@ struct apic_state {
 
     struct guest_info * core;
 
+    void * controller_handle;
+
+    struct v3_timer * timer;
+
     uint32_t eoi;
 
-    v3_lock_t  lock;
+
 };
 
 
@@ -217,15 +235,18 @@ struct apic_state {
 
 struct apic_dev_state {
     int num_apics;
-
+  
     struct apic_state apics[0];
 } __attribute__((packed));
 
 
 
+
+
 static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, uint_t length, void * priv_data);
 static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, uint_t length, void * priv_data);
 
+// No lcoking done
 static void init_apic_state(struct apic_state * apic, uint32_t id) {
     apic->base_addr = DEFAULT_BASE_ADDR;
 
@@ -282,20 +303,21 @@ static void init_apic_state(struct apic_state * apic, uint32_t id) {
     apic->ext_apic_ctrl.val = 0x00000000;
     apic->spec_eoi.val = 0x00000000;
 
-    v3_lock_init(&(apic->lock));
+
 }
 
 
 
 
+
 static int read_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t * dst, void * priv_data) {
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)priv_data;
     struct apic_state * apic = &(apic_dev->apics[core->cpu_id]);
 
     PrintDebug("apic %u: core %u: MSR read\n", apic->lapic_id.val, core->cpu_id);
-    v3_lock(apic->lock);
+
     dst->value = apic->base_addr;
-    v3_unlock(apic->lock);
+
     return 0;
 }
 
@@ -315,7 +337,7 @@ static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, vo
        return -1;
     }
     
-    v3_lock(apic->lock);
+
 
     v3_delete_mem_region(core->vm_info, old_reg);
 
@@ -326,16 +348,20 @@ static int write_apic_msr(struct guest_info * core, uint_t msr, v3_msr_t src, vo
                         apic_read, apic_write, apic_dev) == -1) {
        PrintError("apic %u: core %u: Could not hook new APIC Base address\n",
                   apic->lapic_id.val, core->cpu_id);
-       v3_unlock(apic->lock);
+
        return -1;
     }
 
-    v3_unlock(apic->lock);
+
     return 0;
 }
 
 
 // irq_num is the bit offset into a 256 bit buffer...
+// return values
+//    -1 = error
+//     0 = OK, no interrupt needed now
+//     1 = OK, interrupt needed now
 static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
     int major_offset = (irq_num & ~0x00000007) >> 3;
     int minor_offset = irq_num & 0x00000007;
@@ -344,8 +370,7 @@ static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
     uint8_t flag = 0x1 << minor_offset;
 
 
-
-    if (irq_num <= 15) {
+    if (irq_num <= 15 || irq_num>255) {
        PrintError("apic %u: core %d: Attempting to raise an invalid interrupt: %d\n", 
                   apic->lapic_id.val, apic->core->cpu_id, irq_num);
        return -1;
@@ -356,14 +381,15 @@ static int activate_apic_irq(struct apic_state * apic, uint32_t irq_num) {
 
     if (*req_location & flag) {
        PrintDebug("Interrupt %d  coallescing\n", irq_num);
+       return 0;
     }
 
     if (*en_location & flag) {
        *req_location |= flag;
+       return 1;
     } else {
        PrintDebug("apic %u: core %d: Interrupt  not enabled... %.2x\n", 
                   apic->lapic_id.val, apic->core->cpu_id,*en_location);
-       return 0;
     }
 
     return 0;
@@ -429,7 +455,7 @@ static int apic_do_eoi(struct apic_state * apic) {
        
        *svc_location &= ~flag;
 
-#ifdef CONFIG_CRAY_XT
+#ifdef V3_CONFIG_CRAY_XT
        
        if ((isr_irq == 238) || 
            (isr_irq == 239)) {
@@ -510,8 +536,8 @@ static int activate_internal_irq(struct apic_state * apic, apic_irq_type_t int_t
 static inline int should_deliver_cluster_ipi(struct guest_info * dst_core, 
                                             struct apic_state * dst_apic, uint8_t mda) {
 
-    if         ( ((mda & 0xf0) == (dst_apic->log_dst.dst_log_id & 0xf0)) &&     // (I am in the cluster and
-         ((mda & 0x0f) & (dst_apic->log_dst.dst_log_id & 0x0f)) ) {  //  I am in the set)
+    if         ( ((mda & 0xf0) == (dst_apic->log_dst.dst_log_id & 0xf0)) &&  /* (I am in the cluster and */
+         ((mda & 0x0f) & (dst_apic->log_dst.dst_log_id & 0x0f)) ) {  /*  I am in the set)        */
 
        PrintDebug("apic %u core %u: accepting clustered IRQ (mda 0x%x == log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
@@ -534,12 +560,15 @@ static inline int should_deliver_flat_ipi(struct guest_info * dst_core,
        PrintDebug("apic %u core %u: accepting flat IRQ (mda 0x%x == log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
                   dst_apic->log_dst.dst_log_id);
-      return 1;
+
+       return 1;
+
   } else {
+
        PrintDebug("apic %u core %u: rejecting flat IRQ (mda 0x%x != log_dst 0x%x)\n",
                   dst_apic->lapic_id.val, dst_core->cpu_id, mda, 
                   dst_apic->log_dst.dst_log_id);
-      return 0;
+       return 0;
   }
 }
 
@@ -552,19 +581,21 @@ static int should_deliver_ipi(struct guest_info * dst_core,
     if (dst_apic->dst_fmt.model == 0xf) {
 
        if (mda == 0xff) {
-           // always deliver broadcast
+           /* always deliver broadcast */
            return 1;
        }
 
        return should_deliver_flat_ipi(dst_core, dst_apic, mda);
+
     } else if (dst_apic->dst_fmt.model == 0x0) {
 
        if (mda == 0xff) {
-           // always deliver broadcast
+           /*  always deliver broadcast */
            return 1;
        }
 
        return should_deliver_cluster_ipi(dst_core, dst_apic, mda);
+
     } else {
        PrintError("apic %u core %u: invalid destination format register value 0x%x for logical mode delivery.\n", 
                   dst_apic->lapic_id.val, dst_core->cpu_id, dst_apic->dst_fmt.model);
@@ -573,33 +604,51 @@ static int should_deliver_ipi(struct guest_info * dst_core,
 }
 
 
+// Only the src_apic pointer is used
 static int deliver_ipi(struct apic_state * src_apic, 
                       struct apic_state * dst_apic, 
                       uint32_t vector, uint8_t del_mode) {
 
+
     struct guest_info * dst_core = dst_apic->core;
 
+
     switch (del_mode) {
 
-       case 0:  //fixed
-       case 1: // lowest priority
+       case APIC_FIXED_DELIVERY:  
+       case APIC_LOWEST_DELIVERY: {
+           // lowest priority - 
+           // caller needs to have decided which apic to deliver to!
+
+           int do_xcall;
+
            PrintDebug("delivering IRQ %d to core %u\n", vector, dst_core->cpu_id); 
 
-           activate_apic_irq(dst_apic, vector);
+           do_xcall = activate_apic_irq(dst_apic, vector);
+           
+           if (do_xcall < 0) { 
+               PrintError("Failed to activate apic irq!\n");
+               return -1;
+           }
 
-           if (dst_apic != src_apic) { 
+           if (do_xcall && (dst_apic != src_apic)) { 
                // Assume core # is same as logical processor for now
                // TODO FIX THIS FIX THIS
                // THERE SHOULD BE:  guestapicid->virtualapicid map,
                //                   cpu_id->logical processor map
                //     host maitains logical proc->phsysical proc
-               PrintDebug(" non-local core, forcing it to exit\n"); 
+               PrintDebug(" non-local core with new interrupt, forcing it to exit now\n"); 
 
+#ifdef V3_CONFIG_MULTITHREAD_OS
                v3_interrupt_cpu(dst_core->vm_info, dst_core->cpu_id, 0);
+#else
+               V3_ASSERT(0);
+#endif
            }
 
            break;
-       case 5: { //INIT
+       }
+       case APIC_INIT_DELIVERY: { 
 
            PrintDebug(" INIT delivery to core %u\n", dst_core->cpu_id);
 
@@ -607,7 +656,7 @@ static int deliver_ipi(struct apic_state * src_apic,
 
            // Sanity check
            if (dst_apic->ipi_state != INIT_ST) { 
-               PrintError(" Warning: core %u is not in INIT state (mode = %d), ignored\n",
+               PrintError(" Warning: core %u is not in INIT state (mode = %d), ignored (assuming this is the deassert)\n",
                           dst_core->cpu_id, dst_apic->ipi_state);
                // Only a warning, since INIT INIT SIPI is common
                break;
@@ -626,7 +675,7 @@ static int deliver_ipi(struct apic_state * src_apic,
 
            break;                                                      
        }
-       case 6: { //SIPI
+       case APIC_SIPI_DELIVERY: { 
 
            // Sanity check
            if (dst_apic->ipi_state != SIPI) { 
@@ -657,23 +706,44 @@ static int deliver_ipi(struct apic_state * src_apic,
            // We transition the target core to SIPI state
            dst_core->core_run_state = CORE_RUNNING;  // note: locking should not be needed here
            dst_apic->ipi_state = STARTED;
-
+           
            // As with INIT, we should not need to do anything else
-
+           
            PrintDebug(" SIPI delivery done\n");
-
+           
            break;                                                      
        }
-       case 2: // SMI                  
-       case 3: // reserved                                             
-       case 4: // NMI                                  
-       case 7: // ExtInt
+       case APIC_SMI_DELIVERY: 
+       case APIC_RES1_DELIVERY: // reserved                                            
+       case APIC_NMI_DELIVERY:
+       case APIC_EXTINT_DELIVERY: // ExtInt
        default:
            PrintError("IPI %d delivery is unsupported\n", del_mode); 
            return -1;
     }
-
+    
     return 0;
+    
+}
+
+static struct apic_state * find_physical_apic(struct apic_dev_state *apic_dev, struct int_cmd_reg *icr)
+{
+    int i;
+    
+    if (icr->dst >0 && icr->dst < apic_dev->num_apics) { 
+       // see if it simply is the core id
+       if (apic_dev->apics[icr->dst].lapic_id.val == icr->dst) { 
+           return &(apic_dev->apics[icr->dst]);
+       }
+    }
+
+    for (i=0;i<apic_dev->num_apics;i++) { 
+       if (apic_dev->apics[i].lapic_id.val == icr->dst) { 
+           return &(apic_dev->apics[i]);
+       }
+    }
+    
+    return NULL;
 
 }
 
@@ -683,20 +753,8 @@ static int route_ipi(struct apic_dev_state * apic_dev,
                     struct int_cmd_reg * icr) {
     struct apic_state * dest_apic = NULL;
 
-    PrintDebug("route_ipi: src_apic=%p, icr_data=%p\n", 
-              src_apic, (void *)(addr_t)icr->val);
-
 
-    if ((icr->dst_mode == 0) && (icr->dst >= apic_dev->num_apics)) { 
-       PrintError("route_ipi: Attempted send to unregistered apic id=%u\n", 
-                  icr->dst);
-       return -1;
-    }
-
-    dest_apic =  &(apic_dev->apics[icr->dst]);
-
-
-    PrintDebug("route_ipi: IPI %s %u from apic %p to %s %s %u (icr=0x%llx)\n",
+    PrintDebug("apic: IPI %s %u from apic %p to %s %s %u (icr=0x%llx)\n",
               deliverymode_str[icr->del_mode], 
               icr->vec, 
               src_apic,               
@@ -705,91 +763,152 @@ static int route_ipi(struct apic_dev_state * apic_dev,
               icr->dst,
               icr->val);
 
+
     switch (icr->dst_shorthand) {
 
-       case 0:  // no shorthand
-           if (icr->dst_mode == 0) { 
-               // physical delivery
+       case APIC_SHORTHAND_NONE:  // no shorthand
+           if (icr->dst_mode == APIC_DEST_PHYSICAL) { 
+
+               dest_apic=find_physical_apic(apic_dev,icr);
+               
+               if (dest_apic==NULL) { 
+                   PrintError("apic: Attempted send to unregistered apic id=%u\n", icr->dst);
+                   return -1;
+               }
 
                if (deliver_ipi(src_apic, dest_apic, 
                                icr->vec, icr->del_mode) == -1) {
-                   PrintError("Error: Could not deliver IPI\n");
+                   PrintError("apic: Could not deliver IPI\n");
                    return -1;
                }
 
-           } else {
-               // logical delivery
-               int i;
-               uint8_t mda = icr->dst;
-
-               for (i = 0; i < apic_dev->num_apics; i++) { 
-                    dest_apic = &(apic_dev->apics[i]);
-                    int del_flag = should_deliver_ipi(dest_apic->core, dest_apic, mda);
-                    
-                    if (del_flag == -1) {
-                        PrintError("Error checking delivery mode\n");
-                        return -1;
-                    } else if (del_flag == 1) {
-                       if (deliver_ipi(src_apic, dest_apic, 
+
+               V3_Print("apic: done\n");
+
+
+           } else if (icr->dst_mode == APIC_DEST_LOGICAL) {
+               
+               if (icr->del_mode!=APIC_LOWEST_DELIVERY ) { 
+                   // logical, but not lowest priority
+                   // we immediately trigger
+                   // fixed, smi, reserved, nmi, init, sipi, etc
+                   int i;
+                   
+                   uint8_t mda = icr->dst;
+                   
+                   for (i = 0; i < apic_dev->num_apics; i++) { 
+                       
+                       dest_apic = &(apic_dev->apics[i]);
+                       
+                       int del_flag = should_deliver_ipi(dest_apic->core, dest_apic, mda);
+                       
+                       if (del_flag == -1) {
+                           PrintError("apic: Error checking delivery mode\n");
+                           return -1;
+                       } else if (del_flag == 1) {
+                           if (deliver_ipi(src_apic, dest_apic, 
+                                           icr->vec, icr->del_mode) == -1) {
+                               PrintError("apic: Error: Could not deliver IPI\n");
+                               return -1;
+                           }
+                       }
+                   }
+               } else {  //APIC_LOWEST_DELIVERY
+                   // logical, lowest priority
+                   int i;
+                   struct apic_state * cur_best_apic = NULL;
+                   uint8_t mda = icr->dst;
+                  
+                   for (i = 0; i < apic_dev->num_apics; i++) { 
+                       int del_flag = 0;
+
+                       dest_apic = &(apic_dev->apics[i]);
+                       
+                       del_flag = should_deliver_ipi(dest_apic->core, dest_apic, mda);
+                       
+                       if (del_flag == -1) {
+                           PrintError("apic: Error checking delivery mode\n");
+
+                           return -1;
+                       } else if (del_flag == 1) {
+                           // update priority for lowest priority scan
+                           if (!cur_best_apic) {
+                               cur_best_apic = dest_apic;  
+                           } else if (dest_apic->task_prio.val < cur_best_apic->task_prio.val) {
+                               cur_best_apic = dest_apic;
+                           } 
+                       }                       
+                   }
+
+                   // now we will deliver to the best one if it exists
+                   if (!cur_best_apic) { 
+                       PrintDebug("apic: lowest priority deliver, but no destinations!\n");
+                   } else {
+                       if (deliver_ipi(src_apic, cur_best_apic, 
                                        icr->vec, icr->del_mode) == -1) {
-                           PrintError("Error: Could not deliver IPI\n");
+                           PrintError("apic: Error: Could not deliver IPI\n");
                            return -1;
                        }
+                       //V3_Print("apic: logical, lowest priority delivery to apic %u\n",cur_best_apic->lapic_id.val);
                    }
                }
            }
-           
+
            break;
            
-       case 1:  // self
+       case APIC_SHORTHAND_SELF:  // self
 
-           if (src_apic == NULL) {
-               PrintError("Sending IPI to self from generic IPI sender\n");
+           if (src_apic == NULL) {    /* this is not an apic, but it's trying to send to itself??? */
+               PrintError("apic: Sending IPI to self from generic IPI sender\n");
                break;
            }
 
-           if (icr->dst_mode == 0) { 
+
+
+           if (icr->dst_mode == APIC_DEST_PHYSICAL)  {  /* physical delivery */
                if (deliver_ipi(src_apic, src_apic, icr->vec, icr->del_mode) == -1) {
-                   PrintError("Could not deliver IPI\n");
+                   PrintError("apic: Could not deliver IPI to self (physical)\n");
+                   return -1;
+               }
+           } else if (icr->dst_mode == APIC_DEST_LOGICAL) {  /* logical delivery */
+               PrintError("apic: use of logical delivery in self (untested)\n");
+               if (deliver_ipi(src_apic, src_apic, icr->vec, icr->del_mode) == -1) {
+                   PrintError("apic: Could not deliver IPI to self (logical)\n");
                    return -1;
                }
-           } else {
-               // logical delivery
-               PrintError("use of logical delivery in self is not yet supported.\n");
-               return -1;
            }
+
            break;
            
-       case 2: 
-       case 3: { // all and all-but-me
-           // assuming that logical verus physical doesn't matter
-           // although it is odd that both are used
+       case APIC_SHORTHAND_ALL: 
+       case APIC_SHORTHAND_ALL_BUT_ME: { /* all and all-but-me */
+           /* assuming that logical verus physical doesn't matter
+              although it is odd that both are used */
            int i;
 
            for (i = 0; i < apic_dev->num_apics; i++) { 
                dest_apic = &(apic_dev->apics[i]);
-
-               if ((dest_apic != src_apic) || (icr->dst_shorthand == 2)) { 
+               
+               if ((dest_apic != src_apic) || (icr->dst_shorthand == APIC_SHORTHAND_ALL)) { 
                    if (deliver_ipi(src_apic, dest_apic, icr->vec, icr->del_mode) == -1) {
-                       PrintError("Error: Could not deliver IPI\n");
+                       PrintError("apic: Error: Could not deliver IPI\n");
                        return -1;
                    }
                }
-           }   
+           }
 
            break;
        }
        default:
-           PrintError("Error routing IPI, invalid Mode (%d)\n", icr->dst_shorthand);
+           PrintError("apic: Error routing IPI, invalid Mode (%d)\n", icr->dst_shorthand);
            return -1;
     }
-    
-
     return 0;
 }
 
 
-
+// External function, expected to acquire lock on apic
 static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, uint_t length, void * priv_data) {
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)(priv_data);
     struct apic_state * apic = &(apic_dev->apics[core->cpu_id]);
@@ -804,7 +923,6 @@ static int apic_read(struct guest_info * core, addr_t guest_addr, void * dst, ui
     if (msr->apic_enable == 0) {
        PrintError("apic %u: core %u: Read from APIC address space with disabled APIC, apic msr=0x%llx\n",
                   apic->lapic_id.val, core->cpu_id, apic->base_addr_msr.value);
-
        return -1;
     }
 
@@ -1113,14 +1231,13 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
 
            PrintError("apic %u: core %u: Attempting to write to read only register %p (error)\n", 
                       apic->lapic_id.val, core->cpu_id, (void *)reg_addr);
-           //  return -1;
 
            break;
 
            // Data registers
        case APIC_ID_OFFSET:
-           PrintDebug("apic %u: core %u: my id is being changed to %u\n", 
-                      apic->lapic_id.val, core->cpu_id, op_val);
+           //V3_Print("apic %u: core %u: my id is being changed to %u\n", 
+           //       apic->lapic_id.val, core->cpu_id, op_val);
 
            apic->lapic_id.val = op_val;
            break;
@@ -1213,29 +1330,35 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
 
            // Action Registers
        case EOI_OFFSET:
-           // do eoi
+           // do eoi 
            apic_do_eoi(apic);
            break;
 
-       case INT_CMD_LO_OFFSET:
+       case INT_CMD_LO_OFFSET: {
+           // execute command 
+
+           struct int_cmd_reg tmp_icr;
+
            apic->int_cmd.lo = op_val;
 
-           PrintDebug("apic %u: core %u: sending cmd 0x%llx to apic %u\n", 
-                      apic->lapic_id.val, core->cpu_id,
-                      apic->int_cmd.val, apic->int_cmd.dst);
+           tmp_icr = apic->int_cmd;
 
-           if (route_ipi(apic_dev, apic, &(apic->int_cmd)) == -1) { 
+           //      V3_Print("apic %u: core %u: sending cmd 0x%llx to apic %u\n", 
+           //       apic->lapic_id.val, core->cpu_id,
+           //       apic->int_cmd.val, apic->int_cmd.dst);
+
+           if (route_ipi(apic_dev, apic, &tmp_icr) == -1) { 
                PrintError("IPI Routing failure\n");
                return -1;
            }
-
            break;
-
-       case INT_CMD_HI_OFFSET:
+       }
+       case INT_CMD_HI_OFFSET: {
            apic->int_cmd.hi = op_val;
-           break;
-
+           V3_Print("apic %u: core %u: writing command high=0x%x\n", apic->lapic_id.val, core->cpu_id,apic->int_cmd.hi);
 
+           break;
+       }
        // Unhandled Registers
        case EXT_APIC_CMD_OFFSET:
        case SEOI_OFFSET:
@@ -1249,13 +1372,14 @@ static int apic_write(struct guest_info * core, addr_t guest_addr, void * src, u
     PrintDebug("apic %u: core %u: Write finished\n", apic->lapic_id.val, core->cpu_id);
 
     return length;
+
 }
 
 
 
 /* Interrupt Controller Functions */
 
-// returns 1 if an interrupt is pending, 0 otherwise
+
 static int apic_intr_pending(struct guest_info * core, void * private_data) {
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)(private_data);
     struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
@@ -1272,6 +1396,8 @@ static int apic_intr_pending(struct guest_info * core, void * private_data) {
     return 0;
 }
 
+
+
 static int apic_get_intr_number(struct guest_info * core, void * private_data) {
     struct apic_dev_state * apic_dev = (struct apic_dev_state *)(private_data);
     struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
@@ -1288,38 +1414,49 @@ static int apic_get_intr_number(struct guest_info * core, void * private_data) {
 }
 
 
-int v3_apic_send_ipi(struct v3_vm_info * vm, struct vm_device * dev, 
-                    struct v3_gen_ipi * ipi) {
-    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(dev->private_data);
+
+int v3_apic_send_ipi(struct v3_vm_info * vm, struct v3_gen_ipi * ipi, void * dev_data) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)
+       (((struct vm_device *)dev_data)->private_data);
     struct int_cmd_reg tmp_icr;
 
     // zero out all the fields
     tmp_icr.val = 0;
 
-
     tmp_icr.vec = ipi->vector;
     tmp_icr.del_mode = ipi->mode;
     tmp_icr.dst_mode = ipi->logical;
     tmp_icr.trig_mode = ipi->trigger_mode;
     tmp_icr.dst_shorthand = ipi->dst_shorthand;
     tmp_icr.dst = ipi->dst;
-    
+
 
     return route_ipi(apic_dev, NULL, &tmp_icr);
 }
 
 
-int v3_apic_raise_intr(struct v3_vm_info * vm, struct vm_device * dev, 
-                      uint32_t irq, uint32_t dst) {
-    struct apic_dev_state * apic_dev = (struct apic_dev_state *)(dev->private_data);
+int v3_apic_raise_intr(struct v3_vm_info * vm, uint32_t irq, uint32_t dst, void * dev_data) {
+    struct apic_dev_state * apic_dev = (struct apic_dev_state *)
+       (((struct vm_device*)dev_data)->private_data);
     struct apic_state * apic = &(apic_dev->apics[dst]); 
+    int do_xcall;
 
     PrintDebug("apic %u core ?: raising interrupt IRQ %u (dst = %u).\n", apic->lapic_id.val, irq, dst); 
 
-    activate_apic_irq(apic, irq);
+    do_xcall = activate_apic_irq(apic, irq);
 
-    if (V3_Get_CPU() != dst) {
+    if (do_xcall < 0) { 
+       PrintError("Failed to activate apic irq\n");
+       return -1;
+    }
+    
+    if (do_xcall > 0 && (V3_Get_CPU() != dst)) {
+#ifdef V3_CONFIG_MULTITHREAD_OS
        v3_interrupt_cpu(vm, dst, 0);
+#else
+       V3_ASSERT(0);
+#endif
+
     }
 
     return 0;
@@ -1332,8 +1469,8 @@ static int apic_begin_irq(struct guest_info * core, void * private_data, int irq
     struct apic_state * apic = &(apic_dev->apics[core->cpu_id]); 
     int major_offset = (irq & ~0x00000007) >> 3;
     int minor_offset = irq & 0x00000007;
-    uint8_t * req_location = apic->int_req_reg + major_offset;
-    uint8_t * svc_location = apic->int_svc_reg + major_offset;
+    uint8_t *req_location = apic->int_req_reg + major_offset;
+    uint8_t *svc_location = apic->int_svc_reg + major_offset;
     uint8_t flag = 0x01 << minor_offset;
 
     if (*req_location & flag) {
@@ -1353,7 +1490,9 @@ static int apic_begin_irq(struct guest_info * core, void * private_data, int irq
 
 
 
+
 /* Timer Functions */
+
 static void apic_update_time(struct guest_info * core, 
                             uint64_t cpu_cycles, uint64_t cpu_freq, 
                             void * priv_data) {
@@ -1444,7 +1583,7 @@ static void apic_update_time(struct guest_info * core,
        }
     }
 
-
+    return;
 }
 
 
@@ -1455,36 +1594,46 @@ static struct intr_ctrl_ops intr_ops = {
 };
 
 
-static struct vm_timer_ops timer_ops = {
+static struct v3_timer_ops timer_ops = {
     .update_timer = apic_update_time,
 };
 
 
 
 
-static int apic_free(struct vm_device * dev) {
+static int apic_free(struct apic_dev_state * apic_dev) {
+    int i = 0;
+    struct v3_vm_info * vm = NULL;
+
+    for (i = 0; i < apic_dev->num_apics; i++) {
+       struct apic_state * apic = &(apic_dev->apics[i]);
+       struct guest_info * core = apic->core;
+       
+       vm = core->vm_info;
+
+       v3_remove_intr_controller(core, apic->controller_handle);
+
+       if (apic->timer) {
+           v3_remove_timer(core, apic->timer);
+       }
 
-    /* TODO: This should crosscall to force an unhook on each CPU */
+       // unhook memory
 
-    //   struct apic_state * apic = (struct apic_state *)dev->private_data;
+    }
 
-    v3_unhook_msr(dev->vm, BASE_ADDR_MSR);
+    v3_unhook_msr(vm, BASE_ADDR_MSR);
 
+    V3_Free(apic_dev);
     return 0;
 }
 
 
 static struct v3_device_ops dev_ops = {
-    .free = apic_free,
-    .reset = NULL,
-    .start = NULL,
-    .stop = NULL,
+    .free = (int (*)(void *))apic_free,
 };
 
 
 
-
-
 static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
     char * dev_id = v3_cfg_val(cfg, "ID");
     struct apic_dev_state * apic_dev = NULL;
@@ -1497,10 +1646,11 @@ static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
 
     apic_dev->num_apics = vm->num_cores;
 
-    struct vm_device * dev = v3_allocate_device(dev_id, &dev_ops, apic_dev);
+    struct vm_device * dev = v3_add_device(vm, dev_id, &dev_ops, apic_dev);
 
-    if (v3_attach_device(vm, dev) == -1) {
+    if (dev == NULL) {
        PrintError("apic: Could not attach device %s\n", dev_id);
+       V3_Free(apic_dev);
        return -1;
     }
 
@@ -1513,20 +1663,26 @@ static int apic_init(struct v3_vm_info * vm, v3_cfg_tree_t * cfg) {
 
        init_apic_state(apic, i);
 
-       v3_register_intr_controller(core, &intr_ops, apic_dev);
+       apic->controller_handle = v3_register_intr_controller(core, &intr_ops, apic_dev);
+
+       apic->timer = v3_add_timer(core, &timer_ops, apic_dev);
 
-       v3_add_timer(core, &timer_ops, apic_dev);
+       if (apic->timer == NULL) {
+           PrintError("APIC: Failed to attach timer to core %d\n", i);
+           v3_remove_device(dev);
+           return -1;
+       }
 
        v3_hook_full_mem(vm, core->cpu_id, apic->base_addr, apic->base_addr + PAGE_SIZE_4KB, apic_read, apic_write, apic_dev);
 
        PrintDebug("apic %u: (setup device): done, my id is %u\n", i, apic->lapic_id.val);
     }
 
-#ifdef CONFIG_DEBUG_APIC
+#ifdef V3_CONFIG_DEBUG_APIC
     for (i = 0; i < vm->num_cores; i++) {
        struct apic_state * apic = &(apic_dev->apics[i]);
-       PrintDebug("apic: sanity check: apic %u (at %p) has id %u and msr value %llx\n",
-                  i, apic, apic->lapic_id.val, apic->base_addr_msr.value);
+       PrintDebug("apic: sanity check: apic %u (at %p) has id %u and msr value %llx and core at %p\n",
+                  i, apic, apic->lapic_id.val, apic->base_addr_msr.value,apic->core);
     }
 #endif