Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


build fixes to merge the Palacios configuration parameters with Linux parameters.
[palacios.git] / palacios / src / palacios / vmm_vnet_core.c
index e263735..c3d9603 100644 (file)
 #include <palacios/vmm_lock.h>
 #include <palacios/vmm_queue.h>
 #include <palacios/vmm_sprintf.h>
+#include <palacios/vmm_ethernet.h>
 
-#ifndef CONFIG_DEBUG_VNET
+#ifndef V3_CONFIG_DEBUG_VNET
 #undef PrintDebug
 #define PrintDebug(fmt, args...)
 #endif
 
-
-/* for UDP encapuslation */
-struct eth_header {
-    uchar_t dest[6];
-    uchar_t src[6];
-    uint16_t type;
-}__attribute__((packed));
-
-struct ip_header {
-    uint8_t version: 4;
-    uint8_t hdr_len: 4;
-    uchar_t tos;
-    uint16_t total_len;
-    uint16_t id;
-    uint8_t flags:     3;
-    uint16_t offset: 13;
-    uchar_t ttl;
-    uchar_t proto;
-    uint16_t cksum;
-    uint32_t src_addr;
-    uint32_t dst_addr;
-}__attribute__((packed));
-
-struct udp_header {
-    uint16_t src_port;
-    uint16_t dst_port;
-    uint16_t len;
-    uint16_t csum;//set to zero, disable the xsum
-}__attribute__((packed));
-
-struct udp_link_header {
-    struct eth_header eth_hdr;
-    struct ip_header ip_hdr;
-    struct udp_header udp_hdr;
-}__attribute__((packed));
-/* end with UDP encapuslation structures */
-
-
-
+int v3_net_debug = 0;
 
 struct eth_hdr {
-    uint8_t dst_mac[6];
-    uint8_t src_mac[6];
+    uint8_t dst_mac[ETH_ALEN];
+    uint8_t src_mac[ETH_ALEN];
     uint16_t type; /* indicates layer 3 protocol type */
 } __attribute__((packed));
 
 
 struct vnet_dev {
     int dev_id;
-    uint8_t mac_addr[6];
+    uint8_t mac_addr[ETH_ALEN];
     struct v3_vm_info * vm;
     struct v3_vnet_dev_ops dev_ops;
     void * private_data;
 
-    int rx_disabled;
-    
     struct list_head node;
 } __attribute__((packed));
 
@@ -94,7 +55,8 @@ struct vnet_brg_dev {
     struct v3_vm_info * vm;
     struct v3_vnet_bridge_ops brg_ops;
 
-    int disabled;
+    uint8_t type;
+
     void * private_data;
 } __attribute__((packed));
 
@@ -119,14 +81,20 @@ struct route_list {
 } __attribute__((packed));
 
 
-#define BUF_SIZE 4096
-struct pkts_buf {
-    int start, end;
-    int num; 
-    v3_lock_t lock;
-    struct v3_vnet_pkt pkts[BUF_SIZE];
+struct queue_entry{
+    uint8_t use;
+    struct v3_vnet_pkt pkt;
+    uint8_t * data;
+    uint32_t size_alloc;
 };
 
+#define VNET_QUEUE_SIZE 1024
+struct vnet_queue {
+       struct queue_entry buf[VNET_QUEUE_SIZE];
+       int head, tail;
+       int count;
+       v3_lock_t lock;
+};
 
 static struct {
     struct list_head routes;
@@ -135,61 +103,62 @@ static struct {
     int num_routes;
     int num_devs;
 
-    struct vnet_brg_dev *bridge;
+    struct vnet_brg_dev * bridge;
 
     v3_lock_t lock;
+    struct vnet_stat stats;
+
+    void * pkt_flush_thread;
 
-    uint8_t sidecores; /* 0 -vnet not running on sidecore, > 0, number of extra cores that can be used by VNET */
-    uint64_t cores_map; /* bitmaps for which cores can be used by VNET for sidecore, maxium 64 */
+    struct vnet_queue pkt_q;
 
     struct hashtable * route_cache;
 } vnet_state;
+       
 
-
-
-
-#ifdef CONFIG_DEBUG_VNET
-static inline void mac_to_string(char mac[6], char * buf) {
-    snprintf(buf, 100, "%d:%d:%d:%d:%d:%d", 
+#ifdef V3_CONFIG_DEBUG_VNET
+static inline void mac_to_string(uint8_t * mac, char * buf) {
+    snprintf(buf, 100, "%2x:%2x:%2x:%2x:%2x:%2x", 
             mac[0], mac[1], mac[2],
             mac[3], mac[4], mac[5]);
 }
 
-static void print_route(struct vnet_route_info *route){
+static void print_route(struct v3_vnet_route * route){
     char str[50];
 
-    mac_to_string(route->route_def.src_mac, str);
+    mac_to_string(route->src_mac, str);
     PrintDebug("Src Mac (%s),  src_qual (%d)\n", 
-                       str, route->route_def.src_mac_qual);
-    mac_to_string(route->route_def.dst_mac, str);
+              str, route->src_mac_qual);
+    mac_to_string(route->dst_mac, str);
     PrintDebug("Dst Mac (%s),  dst_qual (%d)\n", 
-                       str, route->route_def.dst_mac_qual);
+              str, route->dst_mac_qual);
     PrintDebug("Src dev id (%d), src type (%d)", 
-                       route->route_def.src_id, 
-                       route->route_def.src_type);
+              route->src_id, 
+              route->src_type);
     PrintDebug("Dst dev id (%d), dst type (%d)\n", 
-                       route->route_def.dst_id, 
-                       route->route_def.dst_type);
-    if (route->route_def.dst_type == LINK_INTERFACE) {
-       PrintDebug("dst_dev (%p), dst_dev_id (%d), dst_dev_ops(%p), dst_dev_data (%p)\n",
-                                       route->dst_dev,
-                                       route->dst_dev->dev_id,
-                                       (void *)&(route->dst_dev->dev_ops),
-                                       route->dst_dev->private_data);
-    }
+              route->dst_id, 
+              route->dst_type);
 }
 
 static void dump_routes(){
-       struct vnet_route_info *route;
+    struct vnet_route_info *route;
 
-       int i = 0;
-       PrintDebug("\n========Dump routes starts ============\n");
-       list_for_each_entry(route, &(vnet_state.routes), node) {
-               PrintDebug("\nroute %d:\n", ++i);
+    int i = 0;
+    PrintDebug("\n========Dump routes starts ============\n");
+    list_for_each_entry(route, &(vnet_state.routes), node) {
+       PrintDebug("\nroute %d:\n", i++);
                
-               print_route(route);
+       print_route(&(route->route_def));
+       if (route->route_def.dst_type == LINK_INTERFACE) {
+           PrintDebug("dst_dev (%p), dst_dev_id (%d), dst_dev_ops(%p), dst_dev_data (%p)\n",
+               route->dst_dev,
+               route->dst_dev->dev_id,
+               (void *)&(route->dst_dev->dev_ops),
+               route->dst_dev->private_data);
        }
-       PrintDebug("\n========Dump routes end ============\n");
+    }
+
+    PrintDebug("\n========Dump routes end ============\n");
 }
 
 #endif
@@ -213,7 +182,7 @@ static int add_route_to_cache(const struct v3_vnet_pkt * pkt, struct route_list
     memcpy(routes->hash_buf, pkt->hash_buf, VNET_HASH_SIZE);    
 
     if (v3_htable_insert(vnet_state.route_cache, (addr_t)routes->hash_buf, (addr_t)routes) == 0) {
-       PrintError("Vnet: Failed to insert new route entry to the cache\n");
+       PrintError("VNET/P Core: Failed to insert new route entry to the cache\n");
        return -1;
     }
     
@@ -221,22 +190,21 @@ static int add_route_to_cache(const struct v3_vnet_pkt * pkt, struct route_list
 }
 
 static int clear_hash_cache() {
-
     v3_free_htable(vnet_state.route_cache, 1, 1);
     vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
 
     return 0;
 }
 
-static int look_into_cache(const struct v3_vnet_pkt * pkt, struct route_list ** routes) {
-    
+static int look_into_cache(const struct v3_vnet_pkt * pkt, 
+                          struct route_list ** routes) {
     *routes = (struct route_list *)v3_htable_search(vnet_state.route_cache, (addr_t)(pkt->hash_buf));
    
     return 0;
 }
 
 
-static struct vnet_dev * find_dev_by_id(int idx) {
+static struct vnet_dev * dev_by_id(int idx) {
     struct vnet_dev * dev = NULL; 
 
     list_for_each_entry(dev, &(vnet_state.devs), node) {
@@ -249,25 +217,29 @@ static struct vnet_dev * find_dev_by_id(int idx) {
     return NULL;
 }
 
-static struct vnet_dev * find_dev_by_mac(char mac[6]) {
+static struct vnet_dev * dev_by_mac(uint8_t * mac) {
     struct vnet_dev * dev = NULL; 
     
     list_for_each_entry(dev, &(vnet_state.devs), node) {
-       if (!memcmp(dev->mac_addr, mac, 6))
+       if (!compare_ethaddr(dev->mac_addr, mac)){
            return dev;
+       }
     }
 
     return NULL;
 }
 
-int v3_vnet_id_by_mac(char mac[6]){
 
-    struct vnet_dev *dev = find_dev_by_mac(mac);
+int v3_vnet_find_dev(uint8_t  * mac) {
+    struct vnet_dev * dev = NULL;
 
-    if (dev == NULL)
-       return -1;
+    dev = dev_by_mac(mac);
 
-    return dev->dev_id;
+    if(dev != NULL) {
+       return dev->dev_id;
+    }
+
+    return -1;
 }
 
 
@@ -278,26 +250,29 @@ int v3_vnet_add_route(struct v3_vnet_route route) {
     new_route = (struct vnet_route_info *)V3_Malloc(sizeof(struct vnet_route_info));
     memset(new_route, 0, sizeof(struct vnet_route_info));
 
-    PrintDebug("Vnet: vnet_add_route_entry: dst_id: %d, dst_type: %d\n",
-                       route.dst_id, route.dst_type);  
+#ifdef V3_CONFIG_DEBUG_VNET
+    PrintDebug("VNET/P Core: add_route_entry:\n");
+    print_route(&route);
+#endif
     
-    memcpy(new_route->route_def.src_mac, route.src_mac, 6);
-    memcpy(new_route->route_def.dst_mac, route.dst_mac, 6);
+    memcpy(new_route->route_def.src_mac, route.src_mac, ETH_ALEN);
+    memcpy(new_route->route_def.dst_mac, route.dst_mac, ETH_ALEN);
     new_route->route_def.src_mac_qual = route.src_mac_qual;
     new_route->route_def.dst_mac_qual = route.dst_mac_qual;
-    new_route->route_def.dst_id = route.dst_id;
     new_route->route_def.dst_type = route.dst_type;
-    new_route->route_def.src_id = route.src_id;
     new_route->route_def.src_type = route.src_type;
+    new_route->route_def.src_id = route.src_id;
+    new_route->route_def.dst_id = route.dst_id;
 
     if (new_route->route_def.dst_type == LINK_INTERFACE) {
-       new_route->dst_dev = find_dev_by_id(new_route->route_def.dst_id);
+       new_route->dst_dev = dev_by_id(new_route->route_def.dst_id);
     }
 
     if (new_route->route_def.src_type == LINK_INTERFACE) {
-       new_route->src_dev = find_dev_by_id(new_route->route_def.src_id);
+       new_route->src_dev = dev_by_id(new_route->route_def.src_id);
     }
 
+
     flags = v3_lock_irqsave(vnet_state.lock);
 
     list_add(&(new_route->node), &(vnet_state.routes));
@@ -306,7 +281,7 @@ int v3_vnet_add_route(struct v3_vnet_route route) {
     v3_unlock_irqrestore(vnet_state.lock, flags);
    
 
-#ifdef CONFIG_DEBUG_VNET
+#ifdef V3_CONFIG_DEBUG_VNET
     dump_routes();
 #endif
 
@@ -314,6 +289,27 @@ int v3_vnet_add_route(struct v3_vnet_route route) {
 }
 
 
+/* delete all route entries with specfied src or dst device id */ 
+static void inline del_routes_by_dev(int dev_id){
+    struct vnet_route_info * route = NULL;
+    unsigned long flags; 
+
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    list_for_each_entry(route, &(vnet_state.routes), node) {
+       if((route->route_def.dst_type == LINK_INTERFACE &&
+            route->route_def.dst_id == dev_id) ||
+            (route->route_def.src_type == LINK_INTERFACE &&
+             route->route_def.src_id == dev_id)){
+             
+           list_del(&(route->node));
+           list_del(&(route->match_node));
+           V3_Free(route);    
+       }
+    }
+
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+}
 
 /* At the end allocate a route_list
  * This list will be inserted into the cache so we don't need to free it
@@ -325,17 +321,17 @@ static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
     int max_rank = 0;
     struct list_head match_list;
     struct eth_hdr * hdr = (struct eth_hdr *)(pkt->data);
-    uint8_t src_type = pkt->src_type;
-    uint32_t src_link = pkt->src_id;
+    //    uint8_t src_type = pkt->src_type;
+    //  uint32_t src_link = pkt->src_id;
 
-#ifdef CONFIG_DEBUG_VNET
+#ifdef V3_CONFIG_DEBUG_VNET
     {
        char dst_str[100];
        char src_str[100];
 
        mac_to_string(hdr->src_mac, src_str);  
        mac_to_string(hdr->dst_mac, dst_str);
-       PrintDebug("Vnet: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
+       PrintDebug("VNET/P Core: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
     }
 #endif
 
@@ -358,14 +354,15 @@ static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
     list_for_each_entry(route, &(vnet_state.routes), node) {
        struct v3_vnet_route * route_def = &(route->route_def);
 
+/*
        // CHECK SOURCE TYPE HERE
        if ( (route_def->src_type != LINK_ANY) && 
             ( (route_def->src_type != src_type) || 
               ( (route_def->src_id != src_link) &&
-                (route_def->src_id != (uint32_t)-1)))) {
+                (route_def->src_id != -1)))) {
            continue;
        }
-
+*/
 
        if ((route_def->dst_mac_qual == MAC_ANY) &&
            (route_def->src_mac_qual == MAC_ANY)) {      
@@ -421,7 +418,7 @@ static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
        }
     }
 
-    PrintDebug("Vnet: match_route: Matches=%d\n", num_matches);
+    PrintDebug("VNET/P Core: match_route: Matches=%d\n", num_matches);
 
     if (num_matches == 0) {
        return NULL;
@@ -442,97 +439,35 @@ static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
     return matches;
 }
 
-static int send_to_bridge(struct v3_vnet_pkt * pkt){
-    struct vnet_brg_dev *bridge = vnet_state.bridge;
-
-    if (bridge == NULL) {
-       PrintError("VNET: No bridge to sent data to links\n");
-       return -1;
-    }
-
-    return bridge->brg_ops.input(bridge->vm, pkt, bridge->private_data);
-}
-
-
-/* enable a vnet device, notify VNET it can send pkts to it */
-int v3_vnet_enable_device(int dev_id){
-    struct vnet_dev *dev = find_dev_by_id(dev_id);
-    unsigned long flags;
-
-    if(!dev)
-       return -1;
-
-    if(!dev->rx_disabled)
-       return 0;
-
-    flags = v3_lock_irqsave(vnet_state.lock);
-    dev->rx_disabled = 0;
-    v3_unlock_irqrestore(vnet_state.lock, flags);
-
-    /* TODO: Wake up all other guests who are trying to send pkts */
-    dev = NULL;
-    list_for_each_entry(dev, &(vnet_state.devs), node) {
-       if (dev->dev_id != dev_id)
-           dev->dev_ops.start_tx(dev->private_data);
-    }
-
-    return 0;
-}
-
-/* Notify VNET to stop sending pkts to it */
-int v3_vnet_disable_device(int dev_id){
-    struct vnet_dev *dev = find_dev_by_id(dev_id);
-    unsigned long flags;
-
-    if(!dev)
-       return -1;
 
-    flags = v3_lock_irqsave(vnet_state.lock);
-    dev->rx_disabled = 1;
-    v3_unlock_irqrestore(vnet_state.lock, flags);
-
-
-    /* TODO: Notify all other guests to stop send pkts */
-    dev = NULL;
-    list_for_each_entry(dev, &(vnet_state.devs), node) {
-       if (dev->dev_id != dev_id)
-           dev->dev_ops.stop_tx(dev->private_data);
-    }
-
-    return 0;
-}
-
-int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
+int vnet_tx_one_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
     struct route_list * matched_routes = NULL;
     unsigned long flags;
     int i;
 
-#ifdef CONFIG_DEBUG_VNET
-   {
-       struct eth_hdr * hdr = (struct eth_hdr *)(pkt->header);
-       char dest_str[100];
-       char src_str[100];
-
-       mac_to_string(hdr->src_mac, src_str);  
-       mac_to_string(hdr->dst_mac, dest_str);
-       int cpu = V3_Get_CPU();
-       PrintDebug("Vnet: on cpu %d, HandleDataOverLink. SRC(%s), DEST(%s), pkt size: %d\n", cpu, src_str, dest_str, pkt->size);
-   }
-#endif
+    int cpu = V3_Get_CPU();
+    V3_Net_Print(2, "VNET/P Core: cpu %d: pkt (size %d, src_id:%d, src_type: %d, dst_id: %d, dst_type: %d)\n",
+                 cpu, pkt->size, pkt->src_id, 
+                 pkt->src_type, pkt->dst_id, pkt->dst_type);
+    if(v3_net_debug >= 4){
+           v3_hexdump(pkt->data, pkt->size, NULL, 0);
+    }
 
     flags = v3_lock_irqsave(vnet_state.lock);
 
+    vnet_state.stats.rx_bytes += pkt->size;
+    vnet_state.stats.rx_pkts++;
+
     look_into_cache(pkt, &matched_routes);
-       
     if (matched_routes == NULL) {  
-       PrintError("Vnet: send pkt Looking into routing table\n");
+       PrintDebug("VNET/P Core: send pkt Looking into routing table\n");
        
        matched_routes = match_route(pkt);
        
        if (matched_routes) {
            add_route_to_cache(pkt, matched_routes);
        } else {
-           PrintDebug("Could not find route for packet... discards packet\n");
+           PrintDebug("VNET/P Core: Could not find route for packet... discards packet\n");
            v3_unlock_irqrestore(vnet_state.lock, flags);
            return 0; /* do we return -1 here?*/
        }
@@ -540,37 +475,107 @@ int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
 
     v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    PrintDebug("Vnet: send pkt route matches %d\n", matched_routes->num_routes);
+    PrintDebug("VNET/P Core: send pkt route matches %d\n", matched_routes->num_routes);
 
     for (i = 0; i < matched_routes->num_routes; i++) {
-        struct vnet_route_info * route = matched_routes->routes[i];
+       struct vnet_route_info * route = matched_routes->routes[i];
        
-        if (route->route_def.dst_type == LINK_EDGE) {                  
-            pkt->dst_type = LINK_EDGE;
-            pkt->dst_id = route->route_def.dst_id;
-
-            if (send_to_bridge(pkt) == -1) {
-                PrintDebug("VNET: Packet not sent properly to bridge\n");
+       if (route->route_def.dst_type == LINK_EDGE) {
+           struct vnet_brg_dev * bridge = vnet_state.bridge;
+           pkt->dst_type = LINK_EDGE;
+           pkt->dst_id = route->route_def.dst_id;
+
+           if (bridge == NULL) {
+               V3_Net_Print(2, "VNET/P Core: No active bridge to sent data to\n");
+                continue;
+           }
+
+           if(bridge->brg_ops.input(bridge->vm, pkt, bridge->private_data) < 0){
+                V3_Net_Print(2, "VNET/P Core: Packet not sent properly to bridge\n");
                 continue;
-            }         
+           }         
+           vnet_state.stats.tx_bytes += pkt->size;
+           vnet_state.stats.tx_pkts ++;
         } else if (route->route_def.dst_type == LINK_INTERFACE) {
-            if (!route->dst_dev->rx_disabled){ 
-                 if(route->dst_dev->dev_ops.input(route->dst_dev->vm, pkt, route->dst_dev->private_data) == -1) {
-                       PrintDebug("VNET: Packet not sent properly\n");
-                       continue;
-                 }
-            }
+            if (route->dst_dev == NULL){
+                 V3_Net_Print(2, "VNET/P Core: No active device to sent data to\n");
+               continue;
+            }
+
+           if(route->dst_dev->dev_ops.input(route->dst_dev->vm, pkt, route->dst_dev->private_data) < 0) {
+                V3_Net_Print(2, "VNET/P Core: Packet not sent properly\n");
+                continue;
+           }
+           vnet_state.stats.tx_bytes += pkt->size;
+           vnet_state.stats.tx_pkts ++;
         } else {
-            PrintError("VNET: Wrong Edge type\n");
+            PrintError("VNET/P Core: Wrong dst type\n");
         }
-
-        PrintDebug("VNET: Forward one packet according to Route %d\n", i);
     }
     
     return 0;
 }
 
-int v3_vnet_add_dev(struct v3_vm_info *vm, uint8_t mac[6], 
+
+static int vnet_pkt_enqueue(struct v3_vnet_pkt * pkt){
+    unsigned long flags;
+    struct queue_entry * entry;
+    struct vnet_queue * q = &(vnet_state.pkt_q);
+    uint16_t num_pages;
+
+    flags = v3_lock_irqsave(q->lock);
+
+    if (q->count >= VNET_QUEUE_SIZE){
+       V3_Net_Print(1, "VNET Queue overflow!\n");
+       v3_unlock_irqrestore(q->lock, flags);
+       return -1;
+    }
+       
+    q->count ++;
+    entry = &(q->buf[q->tail++]);
+    q->tail %= VNET_QUEUE_SIZE;
+       
+    v3_unlock_irqrestore(q->lock, flags);
+
+    /* this is ugly, but should happen very unlikely */
+    while(entry->use);
+
+    if(entry->size_alloc < pkt->size){
+       if(entry->data != NULL){
+           V3_FreePages(V3_PAddr(entry->data), (entry->size_alloc / PAGE_SIZE));
+           entry->data = NULL;
+       }
+
+       num_pages = 1 + (pkt->size / PAGE_SIZE);
+       entry->data = V3_VAddr(V3_AllocPages(num_pages));
+       if(entry->data == NULL){
+           return -1;
+       }
+       entry->size_alloc = PAGE_SIZE * num_pages;
+    }
+
+    entry->pkt.data = entry->data;
+    memcpy(&(entry->pkt), pkt, sizeof(struct v3_vnet_pkt));
+    memcpy(entry->data, pkt->data, pkt->size);
+
+    entry->use = 1;
+
+    return 0;
+}
+
+
+int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data, int synchronize) {
+    if(synchronize){
+       vnet_tx_one_pkt(pkt, NULL);
+    }else {
+       vnet_pkt_enqueue(pkt);
+       V3_Net_Print(2, "VNET/P Core: Put pkt into Queue: pkt size %d\n", pkt->size);
+    }
+       
+    return 0;
+}
+
+int v3_vnet_add_dev(struct v3_vm_info * vm, uint8_t * mac, 
                    struct v3_vnet_dev_ops *ops,
                    void * priv_data){
     struct vnet_dev * new_dev = NULL;
@@ -579,20 +584,19 @@ int v3_vnet_add_dev(struct v3_vm_info *vm, uint8_t mac[6],
     new_dev = (struct vnet_dev *)V3_Malloc(sizeof(struct vnet_dev)); 
 
     if (new_dev == NULL) {
-       PrintError("VNET: Malloc fails\n");
+       PrintError("Malloc fails\n");
        return -1;
     }
    
     memcpy(new_dev->mac_addr, mac, 6);
     new_dev->dev_ops.input = ops->input;
-    new_dev->dev_ops.poll = ops->poll;
     new_dev->private_data = priv_data;
     new_dev->vm = vm;
-    new_dev->dev_id = 0;       
+    new_dev->dev_id = 0;
 
     flags = v3_lock_irqsave(vnet_state.lock);
 
-    if (!find_dev_by_mac(mac)) {
+    if (dev_by_mac(mac) == NULL) {
        list_add(&(new_dev->node), &(vnet_state.devs));
        new_dev->dev_id = ++vnet_state.num_devs;
     }
@@ -601,63 +605,84 @@ int v3_vnet_add_dev(struct v3_vm_info *vm, uint8_t mac[6],
 
     /* if the device was found previosly the id should still be 0 */
     if (new_dev->dev_id == 0) {
-       PrintError("Device Alrady exists\n");
+       PrintError("VNET/P Core: Device Already exists\n");
        return -1;
     }
 
-    PrintDebug("Vnet: Add Device: dev_id %d\n", new_dev->dev_id);
+    PrintDebug("VNET/P Core: Add Device: dev_id %d\n", new_dev->dev_id);
 
     return new_dev->dev_id;
 }
 
 
-/* TODO: Still need to figure out how to handle this multicore part --Lei
-  */
-void  v3_vnet_poll(struct v3_vm_info *vm){
+int v3_vnet_del_dev(int dev_id){
+    struct vnet_dev * dev = NULL;
+    unsigned long flags;
+
+    flags = v3_lock_irqsave(vnet_state.lock);
+       
+    dev = dev_by_id(dev_id);
+    if (dev != NULL){
+       list_del(&(dev->node));
+       del_routes_by_dev(dev_id);
+    }
+       
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+
+    V3_Free(dev);
+
+    PrintDebug("VNET/P Core: Remove Device: dev_id %d\n", dev_id);
+
+    return 0;
+}
+
+
+int v3_vnet_stat(struct vnet_stat * stats){
+       
+    stats->rx_bytes = vnet_state.stats.rx_bytes;
+    stats->rx_pkts = vnet_state.stats.rx_pkts;
+    stats->tx_bytes = vnet_state.stats.tx_bytes;
+    stats->tx_pkts = vnet_state.stats.tx_pkts;
+
+    return 0;
+}
+
+static void free_devices(){
     struct vnet_dev * dev = NULL; 
 
-    switch (vnet_state.sidecores) {
-       case 0:
-               list_for_each_entry(dev, &(vnet_state.devs), node) {
-                   if(dev->vm == vm){
-                       dev->dev_ops.poll(vm, dev->private_data);
-                   }
-              }
-               break;
-       case 1:
-               break;
-       case 2:
-           list_for_each_entry(dev, &(vnet_state.devs), node) {
-               int cpu_id = vm->cores[0].cpu_id + 2; /* temporary here, should use vnet_state.cores_map */
-                struct v3_vnet_dev_xcall_args dev_args; /* could cause problem here -LX */
-                dev_args.vm = vm;
-               dev_args.private_data = dev->private_data;
-               V3_Call_On_CPU(cpu_id, dev->dev_ops.poll_xcall, (void *)&dev_args);
-           }
-           break;
-       default:
-           break;
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       list_del(&(dev->node));
+       V3_Free(dev);
+    }
+}
+
+static void free_routes(){
+    struct vnet_route_info * route = NULL; 
+
+    list_for_each_entry(route, &(vnet_state.routes), node) {
+       list_del(&(route->node));
+       list_del(&(route->match_node));
+       V3_Free(route);
     }
 }
 
 int v3_vnet_add_bridge(struct v3_vm_info * vm,
-                      struct v3_vnet_bridge_ops *ops,
+                      struct v3_vnet_bridge_ops * ops,
+                      uint8_t type,
                       void * priv_data) {
     unsigned long flags;
     int bridge_free = 0;
     struct vnet_brg_dev * tmp_bridge = NULL;    
     
     flags = v3_lock_irqsave(vnet_state.lock);
-
     if (vnet_state.bridge == NULL) {
        bridge_free = 1;
        vnet_state.bridge = (void *)1;
     }
-
     v3_unlock_irqrestore(vnet_state.lock, flags);
 
     if (bridge_free == 0) {
-       PrintError("Bridge already set\n");
+       PrintError("VNET/P Core: Bridge already set\n");
        return -1;
     }
 
@@ -671,10 +696,9 @@ int v3_vnet_add_bridge(struct v3_vm_info * vm,
     
     tmp_bridge->vm = vm;
     tmp_bridge->brg_ops.input = ops->input;
-    tmp_bridge->brg_ops.xcall_input = ops->xcall_input;
-    tmp_bridge->brg_ops.polling_pkt = ops->polling_pkt;
+    tmp_bridge->brg_ops.poll = ops->poll;
     tmp_bridge->private_data = priv_data;
-    tmp_bridge->disabled = 0;
+    tmp_bridge->type = type;
        
     /* make this atomic to avoid possible race conditions */
     flags = v3_lock_irqsave(vnet_state.lock);
@@ -685,36 +709,40 @@ int v3_vnet_add_bridge(struct v3_vm_info * vm,
 }
 
 
-#if 0
-int v3_vnet_disable_bridge() {
-    unsigned long flags; 
-    
-    flags = v3_lock_irqsave(vnet_state.lock);
-
-    if (vnet_state.bridge != NULL) {
-       vnet_state.bridge->disabled = 1;
-    }
+static int vnet_tx_flush(void *args){
+    unsigned long flags;
+    struct queue_entry * entry;
+    struct vnet_queue * q = &(vnet_state.pkt_q);
 
-    v3_unlock_irqrestore(vnet_state.lock, flags);
+    V3_Print("VNET/P Handing Pkt Thread Starting ....\n");
 
-    return 0;
-}
+    //V3_THREAD_SLEEP();
+    /* we need thread sleep/wakeup in Palacios */
+    while(1){
+       flags = v3_lock_irqsave(q->lock);
 
+       if (q->count <= 0){
+           v3_unlock_irqrestore(q->lock, flags);
+           v3_yield(NULL);
+           //V3_THREAD_SLEEP();
+       }else {
+           q->count --;
+           entry = &(q->buf[q->head++]);
+           q->head %= VNET_QUEUE_SIZE;
 
-int v3_vnet_enable_bridge() {
-    unsigned long flags; 
-    
-    flags = v3_lock_irqsave(vnet_state.lock);
+           v3_unlock_irqrestore(q->lock, flags);
 
-    if (vnet_state.bridge != NULL) {
-       vnet_state.bridge->disabled = 0;
-    }
+           /* this is ugly, but should happen very unlikely */
+           while(!entry->use);
+           vnet_tx_one_pkt(&(entry->pkt), NULL);
 
-    v3_unlock_irqrestore(vnet_state.lock, flags);
+           /* asynchronizely release allocated memory for buffer entry here */     
+           entry->use = 0;
 
-    return 0;
+           V3_Net_Print(2, "vnet_tx_flush: pkt (size %d)\n", entry->pkt.size);   
+       }
+    }
 }
-#endif
 
 int v3_init_vnet() {
     memset(&vnet_state, 0, sizeof(vnet_state));
@@ -725,24 +753,35 @@ int v3_init_vnet() {
     vnet_state.num_devs = 0;
     vnet_state.num_routes = 0;
 
-    PrintDebug("VNET: Links and Routes tables initiated\n");
-
     if (v3_lock_init(&(vnet_state.lock)) == -1){
-        PrintError("VNET: Failure to init lock for routes table\n");
+        PrintError("VNET/P Core: Fails to initiate lock\n");
     }
-    PrintDebug("VNET: Locks initiated\n");
 
     vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
-
     if (vnet_state.route_cache == NULL) {
-        PrintError("Vnet: Route Cache Init Fails\n");
+        PrintError("VNET/P Core: Fails to initiate route cache\n");
         return -1;
     }
 
-    vnet_state.sidecores = 0;
-    vnet_state.cores_map = 0;
+    v3_lock_init(&(vnet_state.pkt_q.lock));
 
-    PrintDebug("VNET: initiated\n");
+    vnet_state.pkt_flush_thread = V3_CREATE_THREAD(vnet_tx_flush, NULL, "VNET_Pkts");
+
+    PrintDebug("VNET/P Core is initiated\n");
 
     return 0;
 }
+
+
+void v3_deinit_vnet(){
+
+    v3_lock_deinit(&(vnet_state.lock));
+
+    free_devices();
+    free_routes();
+
+    v3_free_htable(vnet_state.route_cache, 1, 1);
+    V3_Free(vnet_state.bridge);
+}
+
+