Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


Update on VNET
[palacios.git] / palacios / src / palacios / vmm_vnet.c
index 72d4049..e5c17bc 100644 (file)
@@ -8,24 +8,22 @@
  * http://www.v3vee.org
  *
  * Copyright (c) 2009, Lei Xia <lxia@northwestern.edu> 
- * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu> 
- * Copyright (c) 2009, Jack Lange <jarusl@cs.northwestern.edu> 
- * Copyright (c) 2009, Peter Dinda <pdinda@northwestern.edu>
+ * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu>  
  * Copyright (c) 2009, The V3VEE Project <http://www.v3vee.org> 
  * All rights reserved.
  *
  * Author: Lei Xia <lxia@northwestern.edu>
  *        Yuan Tang <ytang@northwestern.edu>
- *        Jack Lange <jarusl@cs.northwestern.edu> 
- *        Peter Dinda <pdinda@northwestern.edu
  *
  * This is free software.  You are permitted to use,
  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  */
  
 #include <palacios/vmm_vnet.h>
-#include <palacios/vmm_hypercall.h>
 #include <palacios/vm_guest_mem.h>
+#include <palacios/vmm_lock.h>
+#include <palacios/vmm_queue.h>
+#include <palacios/vmm_sprintf.h>
 
 #ifndef CONFIG_DEBUG_VNET
 #undef PrintDebug
 #endif
 
 
-struct ethernet_pkt {
-    uint32_t size;
-    uint16_t type;
-    char data[ETHERNET_PACKET_LEN];
-};
 
+struct eth_hdr {
+    uint8_t dst_mac[6];
+    uint8_t src_mac[6];
+    uint16_t type; // indicates layer 3 protocol type
+} __attribute__((packed));
 
-// 14 (ethernet frame) + 20 bytes
-struct in_pkt_header {
-    char ethernetdest[6];
-    char ethernetsrc[6];
-    unsigned char ethernettype[2]; // indicates layer 3 protocol type
-    char ip[20];
-};
 
-#define VNET_INITAB_HCALL 0xca00  // inital hypercall id
 
-#define MAX_LINKS 10
-#define MAX_ROUTES 10
-#define HASH_KEY_LEN 16
-#define MIN_CACHE_SIZE 100
-static const uint_t hash_key_size = 16;
 
-struct link_table {
-    struct link_entry * links[MAX_LINKS];
-    uint16_t size;
-};
 
-struct routing_table {
-    struct routing_entry * routes[MAX_ROUTES];
-    uint16_t size;
+struct vnet_dev {
+
+    uint8_t mac_addr[6];
+    struct v3_vm_info * vm;
+    
+    int (*input)(struct v3_vm_info * vm, struct v3_vnet_pkt * pkt, void * private_data);
+    void * private_data;
+    
+    int dev_id;
+    struct list_head node;
+} __attribute__((packed));
+
+
+#define BRIDGE_BUF_SIZE 1024
+struct bridge_pkts_buf {
+    int start, end;
+    int num; 
+    v3_lock_t lock;
+    struct v3_vnet_pkt pkts[BRIDGE_BUF_SIZE];
+    uint8_t datas[ETHERNET_PACKET_LEN*BRIDGE_BUF_SIZE];
 };
 
-static struct link_table g_links;
-static struct routing_table g_routes;
-static struct gen_queue * g_inpkt_q;
+struct vnet_brg_dev {
+    struct v3_vm_info * vm;
+    
+    int (*input)(struct v3_vm_info * vm, struct v3_vnet_pkt pkt[], uint16_t pkt_num, void * private_data);
+    void (*xcall_input)(void *data);
 
-/* Hash key format:
- * 0-5:     src_eth_addr
- * 6-11:    dest_eth_addr
- * 12:      src type
- * 13-16:   src index
- */
-typedef char * route_hashkey_t;
+    struct bridge_pkts_buf recv_buf;  //packets from Vnet to vnet_bridge device
+
+    struct bridge_pkts_buf send_buf;  //packets from vnet_bridge device to Vnet
 
-// This is the hash value, Format: 0: num_matched_routes, 1...n: matches[] -- TY
-struct route_cache_entry {
-    int num_matched_routes;
-    int * matches; 
+    int disabled;
+       
+    uint16_t max_delayed_pkts;
+    long max_latency; //in cycles
+    void * private_data;
+} __attribute__((packed));
+
+
+
+
+
+struct vnet_route_info {
+    struct v3_vnet_route route_def;
+
+    struct vnet_dev * dst_dev;
+    struct vnet_dev * src_dev;
+
+    struct list_head node;
+    struct list_head match_node; // used for route matching
 };
 
-// the route cache
-static struct hashtable * g_route_cache; 
 
+struct route_list {
+    uint8_t hash_buf[VNET_HASH_SIZE];
+
+    uint32_t num_routes;
+    struct vnet_route_info * routes[0];
+} __attribute__((packed));
 
-static void print_packet(char * pkt, int size) {
-    PrintDebug("Vnet: print_data_packet: size: %d\n", size);
-    v3_hexdump(pkt, size, NULL, 0);
-}
 
-#if 0
-static void print_packet_addr(char * pkt) {
-    PrintDebug("Vnet: print_packet_destination_addr: ");
-    v3_hexdump(pkt + 8, 6, NULL, 0);
+
+static struct {
+    struct list_head routes;
+    struct list_head devs;
     
-    PrintDebug("Vnet: print_packet_source_addr: ");
-    v3_hexdump(pkt + 14, 6, NULL, 0);
-}
+    int num_routes;
+    int num_devs;
 
-static void print_device_addr(char * ethaddr) {
-    PrintDebug("Vnet: print_device_addr: ");
-    v3_hexdump(ethaddr, 6, NULL, 0);
-} 
-#endif
+    struct vnet_brg_dev *bridge;
 
+    v3_lock_t lock;
 
-// network connection functions 
-static inline void ethernet_packet_init(struct ethernet_pkt * pt, const char * data, const size_t size) {
-    pt->size = size;
-    memcpy(pt->data, data, size);
-}
+    struct gen_queue * inpkt_q;
+    struct hashtable * route_cache;
 
-static uint_t hash_from_key_fn(addr_t hashkey) {    
-    uint8_t * key = (uint8_t *)hashkey;
-    return v3_hash_buffer(key, HASH_KEY_LEN);
-}
+} vnet_state;
 
-static int hash_key_equal(addr_t key1, addr_t key2) {
-    uint8_t * buf1 = (uint8_t *)key1;
-    uint8_t * buf2 = (uint8_t *)key2;
-    return (memcmp(buf1, buf2, HASH_KEY_LEN) == 0);
-}
 
-static int init_route_cache() {
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, &hash_from_key_fn, &hash_key_equal);
 
-    if (g_route_cache == NULL) {
-        PrintError("Vnet: Route Cache Initiate Failurely\n");
-        return -1;
-    }
 
-    return 0;
+#ifdef CONFIG_DEBUG_VNET
+static inline void mac_to_string(char mac[6], char * buf) {
+    snprintf(buf, 100, "%d:%d:%d:%d:%d:%d", 
+            mac[0], mac[1], mac[2],
+            mac[3], mac[4], mac[5]);
+}
+
+static void print_route(struct vnet_route_info *route){
+    char str[50];
+
+    mac_to_string(route->route_def.src_mac, str);
+    PrintDebug("Src Mac (%s),  src_qual (%d)\n", 
+                       str, route->route_def.src_mac_qual);
+    mac_to_string(route->route_def.dst_mac, str);
+    PrintDebug("Dst Mac (%s),  dst_qual (%d)\n", 
+                       str, route->route_def.dst_mac_qual);
+    PrintDebug("Src dev id (%d), src type (%d)", 
+                       route->route_def.src_id, 
+                       route->route_def.src_type);
+    PrintDebug("Dst dev id (%d), dst type (%d)\n", 
+                       route->route_def.dst_id, 
+                       route->route_def.dst_type);
+    if (route->route_def.dst_type == LINK_INTERFACE) {
+       PrintDebug("dst_dev (%p), dst_dev_id (%d), dst_dev_input (%p), dst_dev_data (%p)\n",
+                                       route->dst_dev,
+                                       route->dst_dev->dev_id,
+                                       route->dst_dev->input,
+                                       route->dst_dev->private_data);
+    }
+}
+
+static void dump_routes(){
+       struct vnet_route_info *route;
+
+       int i = 0;
+       PrintDebug("\n========Dump routes starts ============\n");
+       list_for_each_entry(route, &(vnet_state.routes), node) {
+               PrintDebug("\nroute %d:\n", ++i);
+               
+               print_route(route);
+       }
+       PrintDebug("\n========Dump routes end ============\n");
 }
 
-static void make_hash_key(route_hashkey_t hashkey,
-                         char src_addr[6],
-                         char dest_addr[6],
-                         char src_type,
-                         int src_index) {
-    int j;
+#endif
 
-    for (j = 0; j < 6; j++) {
-       hashkey[j] = src_addr[j];
-       hashkey[j + 6] = dest_addr[j] + 1;
-    }
 
-    hashkey[12] = src_type;
+/* 
+ * A VNET packet is a packed struct with the hashed fields grouped together.
+ * This means we can generate the hash from an offset into the pkt struct
+ */
+static inline uint_t hash_fn(addr_t hdr_ptr) {    
+    uint8_t * hdr_buf = (uint8_t *)hdr_ptr;
+
+    return v3_hash_buffer(hdr_buf, VNET_HASH_SIZE);
+}
 
-    *(int *)(hashkey + 12) = src_index;
+static inline int hash_eq(addr_t key1, addr_t key2) {  
+    return (memcmp((uint8_t *)key1, (uint8_t *)key2, VNET_HASH_SIZE) == 0);
 }
 
-static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int * matches) {
-    struct route_cache_entry * new_entry = NULL;
-    int i;
-    
-    new_entry = (struct route_cache_entry *)V3_Malloc(sizeof(struct route_cache_entry));
-    if (new_entry == NULL) {
-       PrintError("Vnet: Malloc fails\n");
-       return -1;
-    }
-    
-    new_entry->num_matched_routes = num_matched_r;
+static int add_route_to_cache(const struct v3_vnet_pkt * pkt, struct route_list * routes) {
+    memcpy(routes->hash_buf, pkt->hash_buf, VNET_HASH_SIZE);    
 
-    new_entry->matches = (int *)V3_Malloc(num_matched_r * sizeof(int));
-    
-    if (new_entry->matches == NULL) {
-       PrintError("Vnet: Malloc fails\n");
+    if (v3_htable_insert(vnet_state.route_cache, (addr_t)routes->hash_buf, (addr_t)routes) == 0) {
+       PrintError("Vnet: Failed to insert new route entry to the cache\n");
        return -1;
     }
     
-    for (i = 0; i < num_matched_r; i++) {
-       new_entry->matches[i] = matches[i];
-    }
-    
-    // here, when v3_htable_insert return 0, it means insert fails
-    if (v3_htable_insert(g_route_cache, (addr_t)hashkey, (addr_t)new_entry) == 0) {
-       PrintError("Vnet: Insert new route entry to cache failed\n");
-       V3_Free(new_entry->matches);
-       V3_Free(new_entry);
-    }
-    
     return 0;
 }
 
 static int clear_hash_cache() {
-    v3_free_htable(g_route_cache, 1, 1);
-               
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, hash_from_key_fn, hash_key_equal);
-    
-    if (g_route_cache == NULL) {
-        PrintError("Vnet: Route Cache Create Failurely\n");
-        return -1;
-    }
+
+    v3_free_htable(vnet_state.route_cache, 1, 1);
+    vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
 
     return 0;
 }
 
-static int look_into_cache(route_hashkey_t hashkey, int * matches) {
-    struct route_cache_entry * found = NULL;
-    int n_matches = -1;
-    int i = 0;
+static int look_into_cache(const struct v3_vnet_pkt * pkt, struct route_list ** routes) {
     
-    found = (struct route_cache_entry *)v3_htable_search(g_route_cache, (addr_t)hashkey);
+    *routes = (struct route_list *)v3_htable_search(vnet_state.route_cache, (addr_t)(pkt->hash_buf));
    
-    if (found != NULL) {
-        n_matches = found->num_matched_routes;
+    return 0;
+}
 
-        for (i = 0; i < n_matches; i++) {
-            matches[i] = found->matches[i];
-       }
+
+static struct vnet_dev * find_dev_by_id(int idx) {
+    struct vnet_dev * dev = NULL; 
+    
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       int dev_id = dev->dev_id;
+
+       if (dev_id == idx)
+           return dev;
     }
 
-    return n_matches;
+    return NULL;
 }
 
-static inline uint8_t hex_nybble_to_nybble(const uint8_t hexnybble) {
-    uint8_t x = toupper(hexnybble);
-
-    if (isdigit(x)) {
-       return x - '0';
-    } else {
-       return 10 + (x - 'A');
+static struct vnet_dev * find_dev_by_mac(char mac[6]) {
+    struct vnet_dev * dev = NULL; 
+    
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       if (!memcmp(dev->mac_addr, mac, 6))
+           return dev;
     }
-}
 
-static inline uint8_t hex_byte_to_byte(const uint8_t hexbyte[2]) {
-    return ((hex_nybble_to_nybble(hexbyte[0]) << 4) + 
-           (hex_nybble_to_nybble(hexbyte[1]) & 0xf));
+    return NULL;
 }
 
-static inline void string_to_mac(const char *str, uint8_t mac[6]) {
-    int k;
+int get_device_id_by_mac(char mac[6]){
 
-    for (k = 0; k < 6; k++) {
-       mac[k] = hex_byte_to_byte(&(str[(2 * k) + k]));
-    }
-}
+    struct vnet_dev *dev = find_dev_by_mac(mac);
 
-static inline void mac_to_string(char mac[6], char * buf) {
-    snprintf(buf, 20, "%x:%x:%x:%x:%x:%x", 
-            mac[0], mac[1], mac[2],
-            mac[3], mac[4], mac[5]);
+    if (dev == NULL)
+       return -1;
+
+    return dev->dev_id;
 }
 
-static int add_link_entry(struct link_entry * link) {
-    int idx;
-    
-    for (idx = 0; idx < MAX_LINKS; idx++) {
-       if (g_links.links[idx] == NULL) {
-           g_links.links[idx] = link;
-           g_links.size++;
-           
-           return idx;
-       }
-    }
 
-    PrintError("No available Link entry\n");
-    return -1;
-}
+int v3_vnet_add_route(struct v3_vnet_route route) {
+    struct vnet_route_info * new_route = NULL;
+    unsigned long flags; 
+
+    new_route = (struct vnet_route_info *)V3_Malloc(sizeof(struct vnet_route_info));
+    memset(new_route, 0, sizeof(struct vnet_route_info));
 
-static int add_route_entry(struct routing_entry * route) {
-    int idx;
+    PrintDebug("Vnet: vnet_add_route_entry: dst_id: %d, dst_type: %d\n",
+                       route.dst_id, route.dst_type);  
     
-    for (idx = 0; idx < MAX_ROUTES; idx++) {
-       if (g_routes.routes[idx] == NULL) {
-           g_routes.routes[idx] = route;
-           g_routes.size++;
-           
-           return idx;
-       }
+    memcpy(new_route->route_def.src_mac, route.src_mac, 6);
+    memcpy(new_route->route_def.dst_mac, route.dst_mac, 6);
+    new_route->route_def.src_mac_qual = route.src_mac_qual;
+    new_route->route_def.dst_mac_qual = route.dst_mac_qual;
+    new_route->route_def.dst_id = route.dst_id;
+    new_route->route_def.dst_type = route.dst_type;
+    new_route->route_def.src_id = route.src_id;
+    new_route->route_def.src_type = route.src_type;
+
+    if (new_route->route_def.dst_type == LINK_INTERFACE) {
+       new_route->dst_dev = find_dev_by_id(new_route->route_def.dst_id);
+       PrintDebug("Vnet: Add route, get device: dev_id %d, input : %p, private_data %p\n",
+                       new_route->dst_dev->dev_id, new_route->dst_dev->input, new_route->dst_dev->private_data);
     }
 
-    PrintError("No available route entry\n");
-    return -1;
+    if (new_route->route_def.src_type == LINK_INTERFACE) {
+       new_route->src_dev = find_dev_by_id(new_route->route_def.src_id);
+    }
+
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    list_add(&(new_route->node), &(vnet_state.routes));
+    clear_hash_cache();
+
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+   
+
+#ifdef CONFIG_DEBUG_VNET
+    dump_routes();
+#endif
+
+    return 0;
 }
 
 
-static int vnet_add_route_entry(char src_mac[6],
-                               char dest_mac[6],
-                               int src_mac_qual,
-                               int dest_mac_qual,
-                               int link_idx,
-                               link_type_t link_type,
-                               int src,
-                               link_type_t src_type) {
-    struct routing_entry * new_route = (struct routing_entry *)V3_Malloc(sizeof(struct routing_entry));
-    int idx = -1;
 
-    memset(new_route, 0, sizeof(struct routing_entry));
+// At the end allocate a route_list
+// This list will be inserted into the cache so we don't need to free it
+static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
+    struct vnet_route_info * route = NULL; 
+    struct route_list * matches = NULL;
+    int num_matches = 0;
+    int max_rank = 0;
+    struct list_head match_list;
+    struct eth_hdr * hdr = (struct eth_hdr *)(pkt->data);
+    uint8_t src_type = pkt->src_type;
+    uint32_t src_link = pkt->src_id;
+
+#ifdef CONFIG_DEBUG_VNET
+    {
+       char dst_str[100];
+       char src_str[100];
 
-    if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-        memcpy(new_route->src_mac, src_mac, 6);
+       mac_to_string(hdr->src_mac, src_str);  
+       mac_to_string(hdr->dst_mac, dst_str);
+       PrintDebug("Vnet: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
     }
+#endif
+
+    INIT_LIST_HEAD(&match_list);
+    
+#define UPDATE_MATCHES(rank) do {                              \
+       if (max_rank < (rank)) {                                \
+           max_rank = (rank);                                  \
+           INIT_LIST_HEAD(&match_list);                        \
+                                                               \
+           list_add(&(route->match_node), &match_list);        \
+           num_matches = 1;                                    \
+       } else if (max_rank == (rank)) {                        \
+           list_add(&(route->match_node), &match_list);        \
+           num_matches++;                                      \
+       }                                                       \
+    } while (0)
+    
+
+    list_for_each_entry(route, &(vnet_state.routes), node) {
+       struct v3_vnet_route * route_def = &(route->route_def);
+
+       // CHECK SOURCE TYPE HERE
+       if ( (route_def->src_type != LINK_ANY) && 
+            ( (route_def->src_type != src_type) || 
+              ( (route_def->src_id != src_link) &&
+                (route_def->src_id != (uint32_t)-1)))) {
+           continue;
+       }
+
+
+       if ((route_def->dst_mac_qual == MAC_ANY) &&
+           (route_def->src_mac_qual == MAC_ANY)) {      
+           UPDATE_MATCHES(3);
+       }
+       
+       if (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) {
+           if (route_def->src_mac_qual != MAC_NOT) {
+               if (route_def->dst_mac_qual == MAC_ANY) {
+                   UPDATE_MATCHES(6);
+               } else if (route_def->dst_mac_qual != MAC_NOT &&
+                          memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
+                   UPDATE_MATCHES(8);
+               }
+           }
+       }
            
-    if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-        memcpy(new_route->dest_mac, dest_mac, 6);
-    }
+       if (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
+           if (route_def->dst_mac_qual != MAC_NOT) {
+               if (route_def->src_mac_qual == MAC_ANY) {
+                   UPDATE_MATCHES(6);
+               } else if ((route_def->src_mac_qual != MAC_NOT) && 
+                          (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {
+                   UPDATE_MATCHES(8);
+               }
+           }
+       }
            
-    new_route->src_mac_qual = src_mac_qual;
-    new_route->dest_mac_qual = dest_mac_qual;
-    new_route->link_idx= link_idx;
-    new_route->link_type = link_type;
-    new_route->src_link_idx = src;
-    new_route->src_type = src_type;
-
-    if ((idx = add_route_entry(new_route)) == -1) {
-       PrintError("Could not add route entry\n");
-        return -1;
-    }
-    
-    clear_hash_cache();
-    
-    return idx;
-}
-
-static void * delete_link_entry(int index) {
-    struct link_entry * link = NULL;
-    void * ret = NULL;
-    link_type_t type;
-  
-    if ((index >= MAX_LINKS) || (g_links.links[index] == NULL)) {
-       return NULL;
+       if ((route_def->dst_mac_qual == MAC_NOT) &&
+           (memcmp(route_def->dst_mac, hdr->dst_mac, 6) != 0)) {
+           if (route_def->src_mac_qual == MAC_ANY) {
+               UPDATE_MATCHES(5);
+           } else if ((route_def->src_mac_qual != MAC_NOT) && 
+                      (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {     
+               UPDATE_MATCHES(7);
+           }
+       }
+       
+       if ((route_def->src_mac_qual == MAC_NOT) &&
+           (memcmp(route_def->src_mac, hdr->src_mac, 6) != 0)) {
+           if (route_def->dst_mac_qual == MAC_ANY) {
+               UPDATE_MATCHES(5);
+           } else if ((route_def->dst_mac_qual != MAC_NOT) &&
+                      (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0)) {
+               UPDATE_MATCHES(7);
+           }
+       }
+       
+       // Default route
+       if ( (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) &&
+            (route_def->dst_mac_qual == MAC_NONE)) {
+           UPDATE_MATCHES(4);
+       }
     }
 
-    link = g_links.links[index];
-    type = g_links.links[index]->type;
+    PrintDebug("Vnet: match_route: Matches=%d\n", num_matches);
 
-    if (type == LINK_INTERFACE) {
-       ret = (void *)g_links.links[index]->dst_dev;
-    } else if (type == LINK_EDGE) {
-       ret = (void *)g_links.links[index]->dst_link;
+    if (num_matches == 0) {
+       return NULL;
     }
 
-    g_links.links[index] = NULL;
-    g_links.size--;
+    matches = (struct route_list *)V3_Malloc(sizeof(struct route_list) + 
+                               (sizeof(struct vnet_route_info *) * num_matches));
 
-    V3_Free(link);
-       
-    return ret;
-}
+    matches->num_routes = num_matches;
 
-static int find_route_entry(char src_mac[6], 
-                           char dest_mac[6], 
-                           int src_mac_qual, 
-                           int dest_mac_qual, 
-                           int link_idx, 
-                           link_type_t link_type, 
-                           int src, 
-                           link_type_t src_type) {
-    int i;
-    char temp_src_mac[6];
-    char temp_dest_mac[6];
-  
-    if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-       memcpy(temp_src_mac, src_mac, 6);
-    } else {
-       memset(temp_src_mac, 0, 6);
-    }
-    
-    if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-       memcpy(temp_dest_mac, dest_mac, 6);
-    } else {
-       memset(temp_dest_mac, 0, 6);
+    {
+       int i = 0;
+       list_for_each_entry(route, &match_list, match_node) {
+           matches->routes[i++] = route;
+       }
     }
-    
-    for (i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes.routes[i] != NULL) {
-           if ((memcmp(temp_src_mac, g_routes.routes[i]->src_mac, 6) == 0) && 
-               (memcmp(temp_dest_mac, g_routes.routes[i]->dest_mac, 6) == 0) &&
-               (g_routes.routes[i]->src_mac_qual == src_mac_qual) &&
-               (g_routes.routes[i]->dest_mac_qual == dest_mac_qual)  &&
-               ( (link_type == LINK_ANY) || 
-                 ((link_type == g_routes.routes[i]->link_type) && (g_routes.routes[i]->link_idx == link_idx))) &&
-               ( (src_type == LINK_ANY) || 
-                 ((src_type == g_routes.routes[i]->src_type) && (g_routes.routes[i]->src_link_idx == src)))) {
-               return i;
-           }
-        } 
-     }
-    
-    return -1;
+
+    return matches;
 }
 
-static int delete_route_entry(int index) {
-    struct routing_entry * route;
 
-    if ((index >= MAX_ROUTES) || (g_routes.routes[index] == NULL)) {
-       PrintError("Could not find route entry %d\n", index);
+static int flush_bridge_pkts(struct vnet_brg_dev *bridge){
+    unsigned long flags;
+    int num, start, send;
+    struct v3_vnet_bridge_input_args args;
+    int cpu_id = bridge->vm->cores[0].cpu_id;
+    int current_core = V3_Get_CPU();
+       
+    if (bridge == NULL) {
+       PrintDebug("VNET: No bridge to sent data to links\n");
        return -1;
     }
 
-    route = g_routes.routes[index];
-    g_routes.routes[index] = NULL;
-    g_routes.size--;
+    flags = v3_lock_irqsave(bridge->recv_buf.lock);
+               
+    num = bridge->recv_buf.num;
+    start = bridge->recv_buf.start;
 
-    V3_Free(route);
+    bridge->recv_buf.num -= num;
+    bridge->recv_buf.start += num;
+    bridge->recv_buf.start %= BRIDGE_BUF_SIZE;
+       
+    v3_unlock_irqrestore(bridge->recv_buf.lock, flags);
 
-    clear_hash_cache();
-    
-    return 0;
-}
 
-static int vnet_delete_route_entry_by_addr(char src_mac[6], 
-                                          char dest_mac[6], 
-                                          int src_mac_qual, 
-                                          int dest_mac_qual, 
-                                          int link_idx, 
-                                          link_type_t type, 
-                                          int src, 
-                                          link_type_t src_type) {
-    int index = find_route_entry(src_mac, dest_mac, src_mac_qual, 
-                                dest_mac_qual, link_idx, type, src, src_type);
-    
-    if (index == -1) {
-       PrintError("Error deleting route entry by addr\n");
+    if(bridge->disabled){
+       PrintDebug("VNET: In flush bridge pkts: Bridge is disabled\n");
        return -1;
     }
-    
-    return delete_route_entry(index);
-}
 
-static int match_route(uint8_t * src_mac, 
-                      uint8_t * dst_mac, 
-                      link_type_t src_type, 
-                      int src_index, 
-                      int * matches) { 
-    int values[MAX_ROUTES];
-    int matched_routes[MAX_ROUTES];
-    
-    int num_matches = 0;
-    int i;
-    int max = 0;
-    int no = 0;
-    int exact_match = 0;
-
-    for(i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes.routes[i] != NULL){
-           if ( (g_routes.routes[i]->src_type != LINK_ANY) &&
-                ( (g_routes.routes[i]->src_type != src_type) ||
-                  ( (g_routes.routes[i]->src_link_idx != src_index) &&
-                    (g_routes.routes[i]->src_link_idx != -1)))) {
-               PrintDebug("Vnet: MatchRoute: Source route is on and does not match\n");
-               continue;
-           }
-       
-           if ( (g_routes.routes[i]->dest_mac_qual == MAC_ANY) &&
-                (g_routes.routes[i]->src_mac_qual == MAC_ANY)) {      
+    if(num <= 2 && num > 0){
+       PrintDebug("VNET: In flush bridge pkts: %d\n", num);
+    }
 
-               matched_routes[num_matches] = i;
-               values[num_matches] = 3;
-               num_matches++;
-           }
+    if(num > 0) {
+       PrintDebug("VNET: In flush bridge pkts to bridge, cur_cpu %d, brige_core: %d\n", current_core, cpu_id);
+       if (current_core == cpu_id){
+           if ((start + num) < BRIDGE_BUF_SIZE){
+               bridge->input(bridge->vm, &(bridge->recv_buf.pkts[start]), num, bridge->private_data);
+           }else {
+               bridge->input(bridge->vm, &(bridge->recv_buf.pkts[start]), (BRIDGE_BUF_SIZE - start), bridge->private_data);                            
+               send = num - (BRIDGE_BUF_SIZE - start);
+               bridge->input(bridge->vm, &(bridge->recv_buf.pkts[0]), send, bridge->private_data);
+           }   
+       }else {
+           args.vm = bridge->vm;
+           args.private_data = bridge->private_data;
        
-           if (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) {
-               if (g_routes.routes[i]->src_mac_qual !=  MAC_NOT) {
-                   if (g_routes.routes[i]->dest_mac_qual == MAC_ANY) {
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 6;
-                       
-                       num_matches++;
-                   } else if (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) == 0) {
-                       if (g_routes.routes[i]->dest_mac_qual != MAC_NOT) {   
-                           matched_routes[num_matches] = i;
-                           values[num_matches] = 8;    
-                           exact_match = 1;
-                           num_matches++;
-                       }
-                   }
-               }
-           }
-       
-           if (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) == 0) {
-               if (g_routes.routes[i]->dest_mac_qual != MAC_NOT) {
-                   if (g_routes.routes[i]->src_mac_qual == MAC_ANY) {
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 6;
-                       
-                       num_matches++;
-                   } else if (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) {
-                       if (g_routes.routes[i]->src_mac_qual != MAC_NOT) {
-                           if (exact_match == 0) {
-                               matched_routes[num_matches] = i;
-                               values[num_matches] = 8;
-                               num_matches++;
-                           }
-                       }
-                   }
-               }
+           if ((start + num) < BRIDGE_BUF_SIZE){
+               args.pkt_num = num;
+               args.vnet_pkts = &(bridge->recv_buf.pkts[start]);
+               V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
+           }else {
+               args.pkt_num = BRIDGE_BUF_SIZE - start;
+               args.vnet_pkts = &(bridge->recv_buf.pkts[start]);
+               V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
+                               
+               send = num - (BRIDGE_BUF_SIZE - start);
+               args.pkt_num = send;
+               args.vnet_pkts = &(bridge->recv_buf.pkts[0]);                   
+               V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
            }
+       }
        
-           if ( (g_routes.routes[i]->dest_mac_qual == MAC_NOT) &&
-                (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) != 0)) {
-               if (g_routes.routes[i]->src_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 5;                
-                   num_matches++;    
-               } else if (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) {
-                   if (g_routes.routes[i]->src_mac_qual != MAC_NOT) {      
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 7;                      
-                       num_matches++;
-                   }
-               }
-           }
-           
-           if ( (g_routes.routes[i]->src_mac_qual == MAC_NOT) &&
-                (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) != 0)) {
-               if (g_routes.routes[i]->dest_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 5;        
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) == 0) {
-                   if (g_routes.routes[i]->dest_mac_qual != MAC_NOT) { 
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 7;
-                       num_matches++;
-                   }
-               }
-           }
-       }
-    } // end for
-    
-    for (i = 0; i < MAX_ROUTES; i++) {
-       if ( (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) &&
-            (g_routes.routes[i]->dest_mac_qual == MAC_NONE) &&
-            ( (g_routes.routes[i]->src_type == LINK_ANY) ||
-              ( (g_routes.routes[i]->src_type == src_type) &&
-                ( (g_routes.routes[i]->src_link_idx == src_index) ||
-                  (g_routes.routes[i]->src_link_idx == -1))))) {
-           matched_routes[num_matches] = i;
-           values[num_matches] = 4;
-           PrintDebug("Vnet: MatchRoute: We matched a default route (%d)\n", i);
-           num_matches++;
-       }
+       PrintDebug("VNET: flush bridge pkts %d\n", num);
     }
-    
-    //If many rules have been matched, we choose one which has the highest value rating
-    if (num_matches == 0) {
-       return 0;
-    }
-    
-    for (i = 0; i < num_matches; i++) {
-       if (values[i] > max) {
-           no = 0;
-           max = values[i];
-           matches[no] = matched_routes[i];
-           no++;
-       } else if (values[i] == max) {
-           matches[no] = matched_routes[i];
-           no++;
-       }
-    }
-    
-    return no;
+                       
+    return 0;
 }
 
-#if 0
-// TODO: To be removed
-static int process_udpdata() 
-{
-    struct ethernet_pkt * pt;
 
-    uint32_t dest = 0;
-    uint16_t remote_port = 0;
-    SOCK link_sock = g_udp_sockfd;
-    int length = sizeof(struct ethernet_pkt) - (2 * sizeof(int));   //minus the "size" and "type" 
+static int send_to_bridge(struct v3_vnet_pkt * pkt){
+    struct vnet_brg_dev *bridge = vnet_state.bridge;
+    int cpu_id = bridge->vm->cores[0].cpu_id;
+    struct v3_vnet_bridge_input_args args;
 
-    //run in a loop to get packets from outside network, adding them to the incoming packet queue
-    while (1) {
-       pt = (struct ethernet_pkt *)V3_Malloc(sizeof(struct ethernet_pkt));
+    if (bridge == NULL) {
+       PrintDebug("VNET: No bridge to sent data to links\n");
+       return -1;
+    }
 
-       if (pt == NULL){
-           PrintError("Vnet: process_udp: Malloc fails\n");
-           continue;
-       }
-       
-       PrintDebug("Vnet: route_thread: socket: [%d]. ready to receive from ip [%x], port [%d] or from VMs\n", link_sock, (uint_t)dest, remote_port);
-       pt->size = V3_RecvFrom_IP( link_sock, dest, remote_port, pt->data, length);
-       PrintDebug("Vnet: route_thread: socket: [%d] receive from ip [%x], port [%d]\n", link_sock, (uint_t)dest, remote_port);
-       
-       if (pt->size <= 0) {
-           PrintDebug("Vnet: process_udp: receiving packet from UDP fails\n");
-           V3_Free(pt);
+    if(bridge->max_delayed_pkts <= 1){
+       if(bridge->disabled){
+           PrintDebug("VNET: Bridge diabled\n");
            return -1;
-       }
-       
-       PrintDebug("Vnet: process_udp: get packet\n");
-       print_packet(pt->data, pt->size);
-    }
-}
+      }
 
-// TODO: To be removed
-static int indata_handler( )
-{
-    if (!use_tcp) {
-       process_udpdata();        
+       args.pkt_num = 1;
+       args.vm = bridge->vm;
+       args.vnet_pkts = pkt;
+       args.private_data = bridge->private_data;
+
+       V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
+       PrintDebug("VNET: sent one packet to the bridge\n");
+       return 0;
     }
 
-    return 0;   
-}
+    unsigned long flags;
+    int end, num=0;
+    struct v3_vnet_pkt *buf;
 
-// TODO: To be removed
-static int start_recv_data()
-{
-    if (use_tcp){
-       
-    } else {
-       SOCK udp_data_socket;
-       
-       if ((udp_data_socket = V3_Create_UDP_Socket()) < 0){
-           PrintError("VNET: Can't setup udp socket\n");
-           return -1; 
-       }
-       PrintDebug("Vnet: vnet_setup_udp: get socket: %d\n", udp_data_socket);
-       g_udp_sockfd = udp_data_socket;
-       
-       store_topologies(udp_data_socket);
-       
-       if (V3_Bind_Socket(udp_data_socket, vnet_udp_port) < 0){ 
-           PrintError("VNET: Can't bind socket\n");
-           return -1;
-       }
-       PrintDebug("VNET: vnet_setup_udp: bind socket successful\n");
+    PrintDebug("VNET: send_to_bridge\n");
+
+    flags = v3_lock_irqsave(bridge->recv_buf.lock);
+
+    if(bridge->disabled && bridge->recv_buf.num >= BRIDGE_BUF_SIZE){
+       PrintDebug("Bridge diabled and bridge receive buffer full\n");
+       v3_unlock_irqrestore(bridge->recv_buf.lock, flags);//maybe should move this after copy
+       num = bridge->recv_buf.num;
+       goto exit;
     }
-    
-    V3_CREATE_THREAD(&indata_handler, NULL, "VNET_DATA_HANDLER");
-    return 0;
-}
+           
+    end =      bridge->recv_buf.end;
+    buf = &(bridge->recv_buf.pkts[end]);
 
-static void store_test_topologies(SOCK fd) 
-{
-    int i;
-    int src_mac_qual = MAC_ANY;
-    int dest_mac_qual = MAC_ANY;
-    uint_t dest;
-#ifndef VNET_SERVER
-    dest = (0 | 172 << 24 | 23 << 16 | 1 );
-    PrintDebug("VNET: store_topologies. NOT VNET_SERVER, dest = %x\n", dest);
-#else
-    dest = (0 | 172 << 24 | 23 << 16 | 2 );
-    PrintDebug("VNET: store_topologies. VNET_SERVER, dest = %x\n", dest);
-#endif
+    bridge->recv_buf.num ++;
+    bridge->recv_buf.end ++;
+    bridge->recv_buf.end %= BRIDGE_BUF_SIZE;
 
-    int type = UDP_TYPE;
-    int src = 0;
-    int src_type= LINK_ANY; //ANY_SRC_TYPE
-    int data_port = 22;
-}
+    num = bridge->recv_buf.num;
 
-#endif
+    v3_unlock_irqrestore(bridge->recv_buf.lock, flags);//maybe should move this after copy
 
-static int handle_one_pkt(struct ethernet_pkt * pkt) {
-    int src_link_index = -1;   //the value of src_link_index of udp always is 0
-    int i;
-    char src_mac[6];
-    char dst_mac[6];
 
-    int matches[MAX_ROUTES];
-    int num_matched_routes = 0;
+    buf->size = pkt->size;
+    buf->dst_id = pkt->dst_id;
+    buf->src_id = pkt->src_id;
+    buf->src_type = pkt->src_type;
+    buf->dst_type = pkt->dst_type;
+    memcpy(buf->header, pkt->header, ETHERNET_HEADER_LEN);
+    memcpy(buf->data, pkt->data, pkt->size);
 
-    struct in_pkt_header header;
+exit:  
 
-    char hash_key[hash_key_size];
-  
-    // get the ethernet and ip headers from the packet
-    memcpy((void *)&header, (void *)pkt->data, sizeof(header));
-    memcpy(src_mac, header.ethernetsrc, 6);
-    memcpy(dst_mac, header.ethernetdest, 6);
+    if (num >= bridge->max_delayed_pkts){
+       flush_bridge_pkts(bridge);
+    }
+
+    return 0;
+}
+
+int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
+    struct route_list * matched_routes = NULL;
+    unsigned long flags;
+    int i;
 
 #ifdef CONFIG_DEBUG_VNET
-    char dest_str[18];
-    char src_str[18];
-    
-    mac_to_string(src_mac, src_str);  
-    mac_to_string(dst_mac, dest_str);
-    
-    PrintDebug("Vnet: HandleDataOverLink. SRC(%s), DEST(%s)\n", src_str, dest_str);
+   {
+       struct eth_hdr * hdr = (struct eth_hdr *)(pkt->header);
+       char dest_str[100];
+       char src_str[100];
+
+       mac_to_string(hdr->src_mac, src_str);  
+       mac_to_string(hdr->dst_mac, dest_str);
+       int cpu = V3_Get_CPU();
+       PrintDebug("Vnet: on cpu %d, HandleDataOverLink. SRC(%s), DEST(%s), pkt size: %d\n", cpu, src_str, dest_str, pkt->size);
+   }
 #endif
 
-    // link_edge -> pt->type???
-    make_hash_key(hash_key, src_mac, dst_mac, LINK_EDGE, src_link_index); 
-    
-    num_matched_routes = look_into_cache((route_hashkey_t)hash_key, matches);
-    
-
-    if (num_matched_routes == -1) {  
-       // no match
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-        num_matched_routes = match_route(src_mac, dst_mac, LINK_ANY, src_link_index, matches);
+    look_into_cache(pkt, &matched_routes);
        
-       if (num_matched_routes > 0) {
-           add_route_to_cache(hash_key, num_matched_routes,matches);      
+    if (matched_routes == NULL) {  
+       PrintDebug("Vnet: send pkt Looking into routing table\n");
+       
+       matched_routes = match_route(pkt);
+       
+       if (matched_routes) {
+           add_route_to_cache(pkt, matched_routes);
+       } else {
+           PrintDebug("Could not find route for packet... discards packet\n");
+           v3_unlock_irqrestore(vnet_state.lock, flags);
+           return -1;
        }
     }
-    
-    PrintDebug("Vnet: HandleDataOverLink: Matches=%d\n", num_matched_routes);
-    
-    for (i = 0; i < num_matched_routes; i++) {
-        int route_index = -1;
-        int link_index = -1;
-       int pkt_len = 0;
-       struct link_entry * link = NULL;
 
-        route_index = matches[i];
-       
-        PrintDebug("Vnet: HandleDataOverLink: Forward packet from link according to Route entry %d\n", route_index);
-       
-       link_index = g_routes.routes[route_index]->link_idx;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-       if ( (link_index < 0) || 
-            (link_index > MAX_LINKS) || 
-            (g_links.links[link_index] == NULL)) {
-           continue;
-       }
-       
-       link = g_links.links[link_index];
+    PrintDebug("Vnet: send pkt route matches %d\n", matched_routes->num_routes);
 
+    for (i = 0; i < matched_routes->num_routes; i++) {
+        struct vnet_route_info * route = matched_routes->routes[i];
        
-       pkt_len = pkt->size;
-
-        if (g_routes.routes[route_index]->link_type == LINK_EDGE) {
-           
-            // TODO: apply the header in the beginning of the packet to be sent
-           if ((link->dst_link->input(pkt->data, pkt_len, NULL)) != pkt_len) {
-               PrintError("LEI --  WHAT IS THE ERROR HERE?\n");
-               return -1;
-           }
-        } else if (g_routes.routes[route_index]->link_type == LINK_INTERFACE) {
-            if ((link->dst_link->input(pkt->data, pkt_len, NULL)) != pkt_len) {
-               PrintError("LEI --  WHAT IS THE ERROR HERE?\n");
-               return -1;
-           }
+        if (route->route_def.dst_type == LINK_EDGE) {
+            pkt->dst_type = LINK_EDGE;
+            pkt->dst_id = route->route_def.dst_id;
+
+            if (send_to_bridge(pkt) == -1) {
+                PrintDebug("VNET: Packet not sent properly to bridge\n");
+                continue;
+            }
+            
+        } else if (route->route_def.dst_type == LINK_INTERFACE) {
+            if (route->dst_dev->input(route->dst_dev->vm, pkt, route->dst_dev->private_data) == -1) {
+                PrintDebug("VNET: Packet not sent properly\n");
+                continue;
+            }
         } else {
-            PrintError("Vnet: Wrong Edge type\n");
-           return -1;
+            PrintDebug("Vnet: Wrong Edge type\n");
+            continue;
         }
+
+        PrintDebug("Vnet: v3_vnet_send_pkt: Forward packet according to Route %d\n", i);
     }
     
     return 0;
 }
 
-static int send_ethernet_pkt(char * buf, int length) {
-    struct ethernet_pkt * pkt;
+void v3_vnet_send_pkt_xcall(void * data){
+    struct v3_vnet_pkt * pkt = (struct v3_vnet_pkt *)data;
+    v3_vnet_send_pkt(pkt, NULL);
+}
 
-    pkt = (struct ethernet_pkt *)V3_Malloc(sizeof(struct ethernet_pkt));
-    ethernet_packet_init(pkt, buf, length);  //====here we copy sending data once 
-       
-    PrintDebug("VNET: vm_send_pkt: transmitting packet: (size:%d)\n", (int)pkt->size);
-    print_packet((char *)buf, length);
-    
-    v3_enqueue(g_inpkt_q, (addr_t)pkt);
 
-    return 0;
-}
+void v3_vnet_bridge_polling()
+{
+    unsigned long flags;
+    int num, start;
+    struct v3_vnet_pkt *buf;
+    struct vnet_brg_dev *bridge = vnet_state.bridge;
 
-int v3_vnet_send_pkt(uchar_t * buf, int length) {
-    PrintDebug("VNET: In V3_Send_pkt: pkt length %d\n", length);
-    
-    return send_ethernet_pkt((char *)buf, length);
-}
+    PrintDebug("In vnet bridge pollling: cpu %d\n", V3_Get_CPU());
 
-static int search_device(char * device_name) {
-    int i;
+    if(bridge == NULL){
+       PrintDebug("VNET: Bridge is not set\n");
+       return;
+    }
 
-    for (i = 0; i < MAX_LINKS; i++) {
-        if ((g_links.links[i] != NULL) && (g_links.links[i]->type == LINK_INTERFACE)) {
-           if (strcmp(device_name, g_links.links[i]->dst_dev->name) == 0) {
-               return i;
-           }
-        }
+    flags = v3_lock_irqsave(bridge->send_buf.lock);
+               
+    num = bridge->send_buf.num;
+    start = bridge->send_buf.start;
+
+    PrintDebug("VNET: bridge polling pkts %d\n", num);
+
+    while(num > 0) {
+       buf = &(bridge->send_buf.pkts[bridge->send_buf.start]);
+
+       v3_vnet_send_pkt(buf, NULL);
+
+       bridge->send_buf.num --;
+       bridge->send_buf.start ++;
+       bridge->send_buf.start %= BRIDGE_BUF_SIZE;
+       num --;
     }
-    
-    PrintError("LEI --  WHAT IS THE ERROR HERE?\n");
-    return -1;
-}
 
-static int vnet_register_device(struct vm_device * vdev, 
-                        char * dev_name, 
-                        uchar_t mac[6], 
-                        int (*netif_input)(uchar_t * pkt, uint_t size, void * private_data), 
-                        void * data) {
-    struct vnet_if_device * if_dev;
+    v3_unlock_irqrestore(bridge->send_buf.lock, flags);
+
+    return;
+}
 
-    int idx = search_device(dev_name);
 
-    if (idx != -1) {
-       PrintError("LEI --  WHAT IS THE ERROR HERE?\n");
+int v3_vnet_bridge_rx(uchar_t *buf, uint16_t size, uint16_t src_link){
+    struct vnet_brg_dev *bridge = vnet_state.bridge;
+    unsigned long flags;
+    int end;
+    struct v3_vnet_pkt *pkt;
+   
+    if (bridge == NULL) {
+       PrintDebug("VNET: No bridge is set\n");
        return -1;
     }
-    
-    if_dev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device));
-    
-    if (if_dev == NULL) {
-       PrintError("VNET: Malloc fails\n");
-       return -1;
+
+    flags = v3_lock_irqsave(bridge->send_buf.lock);
+           
+    end =      bridge->send_buf.end;
+    pkt = &(bridge->send_buf.pkts[end]);
+
+    if(bridge->send_buf.num > BRIDGE_BUF_SIZE){
+       PrintDebug("VNET: bridge rx: buffer full\n");
+       goto exit;
     }
-    
-    strcpy(if_dev->name, dev_name);
-    strncpy(if_dev->mac_addr, mac, 6);
-    if_dev->dev = vdev;
-    if_dev->input = netif_input;
-    if_dev->private_data = data;
 
-    struct link_entry * link = (struct link_entry *)V3_Malloc(sizeof(struct link_entry));
+    bridge->send_buf.num ++;
+    bridge->send_buf.end ++;
+    bridge->send_buf.end %= BRIDGE_BUF_SIZE;
 
-    link->type = LINK_INTERFACE;
-    link->dst_dev = if_dev;
+    pkt->size = size;
+    pkt->src_id = src_link;
+    pkt->src_type = LINK_EDGE;
+    memcpy(pkt->header, buf, ETHERNET_HEADER_LEN);
+    memcpy(pkt->data, buf, size);
 
-    idx = add_link_entry(link);
-    
-    return idx;
+exit:
+       
+    v3_unlock_irqrestore(bridge->send_buf.lock, flags);
+
+    return 0;
 }
+       
 
-static int vnet_unregister_device(char * dev_name) {
-    int idx;
+int v3_vnet_add_dev(struct v3_vm_info *vm, uint8_t mac[6], 
+                   int (*netif_input)(struct v3_vm_info * vm, struct v3_vnet_pkt * pkt, void * private_data), 
+                   void * priv_data){
+    struct vnet_dev * new_dev = NULL;
+    unsigned long flags;
 
-    idx = search_device(dev_name);
-    
-    if (idx == -1) {
-       PrintError("LEI --  WHAT IS THE ERROR HERE?\n");
-        return -1;
-    }
+    new_dev = (struct vnet_dev *)V3_Malloc(sizeof(struct vnet_dev)); 
 
-    struct vnet_if_device * device = (struct vnet_if_device *)delete_link_entry(idx);
-    if (device == NULL) {
-       PrintError("LEI --  WHAT IS THE ERROR HERE?\n");
+    if (new_dev == NULL) {
+       PrintError("VNET: Malloc fails\n");
        return -1;
     }
+   
+    memcpy(new_dev->mac_addr, mac, 6);
+    new_dev->input = netif_input;
+    new_dev->private_data = priv_data;
+    new_dev->vm = vm;
+    new_dev->dev_id = 0;       
 
-    V3_Free(device);
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    return idx;
-}
+    if (!find_dev_by_mac(mac)) {
+       list_add(&(new_dev->node), &(vnet_state.devs));
+       new_dev->dev_id = ++vnet_state.num_devs;
+    }
 
-int v3_vnet_pkt_process() {
-    struct ethernet_pkt * pkt;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    PrintDebug("VNET: In vnet_check\n");
-       
-    while ((pkt = (struct ethernet_pkt *)v3_dequeue(g_inpkt_q)) != NULL) {
-       PrintDebug("VNET: In vnet_check: pt length %d, pt type %d\n", (int)pkt->size, (int)pkt->type);
-       v3_hexdump(pkt->data, pkt->size, NULL, 0);
-       
-       if (handle_one_pkt(pkt)) {
-           PrintDebug("VNET: vnet_check: handle one packet!\n");  
-       } else {
-           PrintError("VNET: vnet_check: fail to forward one packet, discard it!\n"); 
-       }
-       
-       V3_Free(pkt); // be careful here
+    // if the device was found previosly the id should still be 0
+    if (new_dev->dev_id == 0) {
+       PrintError("Device Alrady exists\n");
+       return -1;
     }
-    
-    return 0;
+
+    PrintDebug("Vnet: Add Device: dev_id %d, input : %p, private_data %p\n",
+                       new_dev->dev_id, new_dev->input, new_dev->private_data);
+
+    return new_dev->dev_id;
 }
 
 
-static void init_empty_link_table() {
-    int i;
+void  v3_vnet_heartbeat(struct guest_info *core){
+    static long last_time, cur_time;
+
+    if(vnet_state.bridge == NULL)
+       return;
+       
+    if(vnet_state.bridge->max_delayed_pkts <= 1)
+       return;
 
-    for (i = 0; i < MAX_LINKS; i++) {
-        g_links.links[i] = NULL;
+    if(V3_Get_CPU() != vnet_state.bridge->vm->cores[0].cpu_id){
+       rdtscll(cur_time);
     }
 
-    g_links.size = 0;
+    if ((cur_time - last_time) >= vnet_state.bridge->max_latency) {
+       last_time = cur_time;
+       flush_bridge_pkts(vnet_state.bridge);
+    }
 }
 
-static void init_empty_route_table() { 
+int v3_vnet_add_bridge(struct v3_vm_info * vm,
+                      int (*input)(struct v3_vm_info * vm, struct v3_vnet_pkt pkt[], uint16_t pkt_num, void * private_data),
+                      void (*xcall_input)(void *data),
+                      uint16_t max_delayed_pkts,
+                      long max_latency,
+                      void * priv_data) {
+    unsigned long flags;
+    int bridge_free = 0;
+    struct vnet_brg_dev * tmp_bridge = NULL;
     int i;
+    
+    
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    for (i = 0; i < MAX_ROUTES; i++) {
-        g_routes.routes[i] = NULL;
+    if (vnet_state.bridge == NULL) {
+       bridge_free = 1;
+       vnet_state.bridge = (void *)1;
     }
 
-    g_links.size = 0;
-}
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-static void init_tables() {
-    init_empty_link_table();
-    init_empty_route_table();
-    init_route_cache();
-}
+    if (bridge_free == 0) {
+       PrintError("Bridge already set\n");
+       return -1;
+    }
 
-static void init_pkt_queue() {
-    PrintDebug("VNET Init package receiving queue\n");
+    tmp_bridge = (struct vnet_brg_dev *)V3_Malloc(sizeof(struct vnet_brg_dev));
 
-    g_inpkt_q = v3_create_queue();
-    v3_init_queue(g_inpkt_q);
-}
+    if (tmp_bridge == NULL) {
+       PrintError("Malloc Fails\n");
+       return -1;
+    }
+    
+    tmp_bridge->vm = vm;
+    tmp_bridge->input = input;
+    tmp_bridge->xcall_input = xcall_input;
+    tmp_bridge->private_data = priv_data;
+    tmp_bridge->disabled = 0;
 
-#if 0
-// TODO: 
-static int init_routing_tables(struct routing_entry *route_tab, uint16_t size)
-{
-    //struct routing_entry *route;
+    //initial receving buffer
+    tmp_bridge->recv_buf.start = 0;
+    tmp_bridge->recv_buf.end = 0;
+    tmp_bridge->recv_buf.num = 0;
+    if(v3_lock_init(&(tmp_bridge->recv_buf.lock)) == -1){
+       PrintError("VNET: add bridge, error to initiate recv buf lock\n");
+    }
+    tmp_bridge->max_delayed_pkts = (max_delayed_pkts<BRIDGE_BUF_SIZE)?max_delayed_pkts : BRIDGE_BUF_SIZE;
+    tmp_bridge->max_latency = max_latency;
+    for(i = 0; i<BRIDGE_BUF_SIZE; i++){
+       tmp_bridge->recv_buf.pkts[i].data = &(tmp_bridge->recv_buf.datas[i*ETHERNET_PACKET_LEN]);
+    }
 
+    //initial sending buffer
+    tmp_bridge->send_buf.start = 0;
+    tmp_bridge->send_buf.end = 0;
+    tmp_bridge->send_buf.num = 0;
+    if(v3_lock_init(&(tmp_bridge->send_buf.lock)) == -1){
+       PrintError("VNET: add bridge, error to initiate send buf lock\n");
+    }
+    for(i = 0; i<BRIDGE_BUF_SIZE; i++){
+       tmp_bridge->send_buf.pkts[i].data = &(tmp_bridge->send_buf.datas[i*ETHERNET_PACKET_LEN]);
+    }
+       
+    // make this atomic to avoid possible race conditions
+    flags = v3_lock_irqsave(vnet_state.lock);
+    vnet_state.bridge = tmp_bridge;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
     return 0;
 }
 
 
-// TODO: 
-static int init_link_tables(struct link_entry *link_tab, uint16_t size)
-{
-    //struct link_entry *link;
+int v3_vnet_disable_bridge() {
+    unsigned long flags; 
+    
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    if (vnet_state.bridge != NULL) {
+       vnet_state.bridge->disabled = 1;
+    }
+
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
     return 0;
 }
 
-#endif
 
-struct table_init_info {
-    addr_t routing_table_start;
-    uint16_t routing_table_size;
-    addr_t link_table_start;
-    uint16_t link_table_size;
-};
-
-static int handle_init_tables_hcall(struct guest_info * info, uint_t hcall_id, void * priv_data) {
-    uint8_t * buf = NULL;
-    addr_t info_addr = (addr_t)info->vm_regs.rcx;
+int v3_vnet_enable_bridge() {
+    unsigned long flags; 
+    
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    if (guest_va_to_host_va(info, info_addr, (addr_t *)&(buf)) == -1) {
-       PrintError("Could not translate buffer address\n");
-       return -1;
+    if (vnet_state.bridge != NULL) {
+       vnet_state.bridge->disabled = 0;
     }
 
-    //struct table_init_info *init_info = (struct table_init_info *)buf;
-    
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+
     return 0;
 }
 
 
-void v3_vnet_init(struct guest_info * vm) {
-    init_tables();
-    init_pkt_queue();
+
+int V3_init_vnet() {
        
-    v3_register_hypercall(vm, VNET_INITAB_HCALL, handle_init_tables_hcall, NULL);
+    INIT_LIST_HEAD(&(vnet_state.routes));
+    INIT_LIST_HEAD(&(vnet_state.devs));
 
-    //store_test_topologies(udp_data_socket); 
+    vnet_state.num_devs = 0;
+    vnet_state.num_routes = 0;
 
-    PrintDebug("VNET Initialized\n");
-}
+    PrintDebug("VNET: Links and Routes tables initiated\n");
 
+    if (v3_lock_init(&(vnet_state.lock)) == -1){
+        PrintError("VNET: Failure to init lock for routes table\n");
+    }
+
+    PrintDebug("VNET: Locks initiated\n");
+
+    vnet_state.inpkt_q = v3_create_queue();
+    v3_init_queue(vnet_state.inpkt_q);
+    PrintDebug("VNET: Receiving queue initiated\n");
+
+    vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
+
+    if (vnet_state.route_cache == NULL) {
+        PrintError("Vnet: Route Cache Init Fails\n");
+        return -1;
+    }
+
+    PrintDebug("VNET: initiated\n");
 
+    return 0;
+}