Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


some fixes to caching, still segfaults at the init process
[palacios.git] / palacios / src / palacios / vmm_vnet.c
index 02f5182..d7455ac 100644 (file)
  * http://www.v3vee.org
  *
  * Copyright (c) 2009, Lei Xia <lxia@northwestern.edu> 
- * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu> 
- * Copyright (c) 2009, Jack Lange <jarusl@cs.northwestern.edu> 
- * Copyright (c) 2009, Peter Dinda <pdinda@northwestern.edu>
+ * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu>  
  * Copyright (c) 2009, The V3VEE Project <http://www.v3vee.org> 
  * All rights reserved.
  *
  * Author: Lei Xia <lxia@northwestern.edu>
  *        Yuan Tang <ytang@northwestern.edu>
- *        Jack Lange <jarusl@cs.northwestern.edu> 
- *        Peter Dinda <pdinda@northwestern.edu
  *
  * This is free software.  You are permitted to use,
  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  */
  
 #include <palacios/vmm_vnet.h>
+#include <palacios/vm_guest_mem.h>
+#include <palacios/vmm_lock.h>
+#include <palacios/vmm_queue.h>
+#include <palacios/vmm_sprintf.h>
+
+#ifndef CONFIG_DEBUG_VNET
+#undef PrintDebug
+#define PrintDebug(fmt, args...)
+#endif
 
-static const char any_type_str[] = "any";
-static const char not_type_str[] = "not";
-static const char none_type_str[] = "none";
-static const char empty_type_str[] = "empty";
-
-static const char link_interface_str[] = "INTERFACE";
-static const char link_edge_str[] = "EDGE";
-static const char link_any_str[] = "ANY";
-
-static const char link_prot_tcp_str[] = "tcp";
-static const char link_prot_udp_str[] = "udp";
 
-typedef enum {MAC_ANY, MAC_NOT, MAC_NONE, MAC_EMPTY} mac_type_t; //for 'src_mac_qual' and 'dst_mac_qual'
-typedef enum {LINK_INTERFACE, LINK_EDGE, LINK_ANY} link_type_t; //for 'type' and 'src_type' in struct routing
-typedef enum {TCP_TYPE, UDP_TYPE} link_prot_type_t;
 
-static const uint_t hash_key_size = 16;
-static SOCK g_udp_sockfd;
-static struct gen_queue * g_inpkt_q;
-static bool use_tcp = false;
-static uint_t vnet_udp_port = 1022;
+struct eth_hdr {
+    uint8_t dst_mac[6];
+    uint8_t src_mac[6];
+    uint16_t type; // indicates layer 3 protocol type
+} __attribute__((packed));
 
-struct raw_ethernet_pkt {
-  int  size;
-  int type; // vm or link type:  INTERFACE|EDGE
-  char  data[ETHERNET_PACKET_LEN];
-};
 
-//static char *vnet_version = "0.9";
-//static int vnet_server = 0;
 
 
-#define MAX_LINKS 1
-#define MAX_ROUTES 1
-#define MAX_DEVICES 16
 
-static struct topology g_links[MAX_LINKS];
-static int g_num_links; //The current number of links
-static int g_first_link;
-static int g_last_link;
+struct vnet_dev {
 
-static struct routing g_routes[MAX_ROUTES];
-static int g_num_routes; //The current number of routes
-static int g_first_route;
-static int g_last_route;
-
-static struct device_list g_devices[MAX_DEVICES];
-static int g_num_devices;
-static int g_first_device;
-static int g_last_device;
+    uint8_t mac_addr[6];
+    struct v3_vm_info * vm;
+    
+    int (*input)(struct v3_vm_info * vm, struct v3_vnet_pkt * pkt, void * private_data);
+    void * private_data;
+    
+    int dev_id;
+    struct list_head node;
+} __attribute__((packed));
 
 
-static void print_packet(char *pkt, int size) {
-    PrintDebug("Vnet: print_data_packet: size: %d\n", size);
-    v3_hexdump(pkt, size, NULL, 0);
-}
+#define BRIDGE_BUF_SIZE 512
+struct bridge_pkts_buf {
+    int start, end;
+    int num; 
+    v3_lock_t lock;
+    struct v3_vnet_pkt pkts[BRIDGE_BUF_SIZE];
+    uint8_t datas[ETHERNET_PACKET_LEN * BRIDGE_BUF_SIZE];
+};
 
-#if 0
-static void print_packet_addr(char *pkt) {
-    PrintDebug("Vnet: print_packet_destination_addr: ");
-    v3_hexdump(pkt + 8, 6, NULL, 0);
+struct vnet_brg_dev {
+    struct v3_vm_info * vm;
     
-    PrintDebug("Vnet: print_packet_source_addr: ");
-    v3_hexdump(pkt + 14, 6, NULL, 0);
-}
+    int (*input)(struct v3_vm_info * vm, struct v3_vnet_pkt pkt[], uint16_t pkt_num, void * private_data);
+    void (*xcall_input)(void * data);
+    int (*polling_pkt)(struct v3_vm_info * vm,  void * private_data);
 
-static void print_device_addr(char *ethaddr) {
-    PrintDebug("Vnet: print_device_addr: ");
-    v3_hexdump(ethaddr, 6, NULL, 0);
-} 
-#endif
+    int disabled;
+       
+    uint16_t max_delayed_pkts;
+    long max_latency; //in cycles
+    void * private_data;
+} __attribute__((packed));
 
 
-//network connection functions 
-static inline void raw_ethernet_packet_init(struct raw_ethernet_pkt * pt, const char * data, const size_t size) {
-    pt->size = size;
-    memcpy(pt->data, data, size);
-}
 
 
 
-/* Hash key format:
- * 0-5:     src_eth_addr
- * 6-11:    dest_eth_addr
- * 12:      src type
- * 13-16:   src index
- */
-typedef char * route_hashkey_t;
+struct vnet_route_info {
+    struct v3_vnet_route route_def;
+
+    struct vnet_dev * dst_dev;
+    struct vnet_dev * src_dev;
 
-// This is the hash value, Format: 0: num_matched_routes, 1...n: matches[] -- TY
-struct route_cache_entry {
-    int num_matched_routes;
-    int * matches; 
+    struct list_head node;
+    struct list_head match_node; // used for route matching
 };
 
-#define HASH_KEY_LEN 16
-#define MIN_CACHE_SIZE 100
 
+struct route_list {
+    uint8_t hash_buf[VNET_HASH_SIZE];
 
-//Header of the route cache
-static struct hashtable * g_route_cache; 
+    uint32_t num_routes;
+    struct vnet_route_info * routes[0];
+} __attribute__((packed));
 
-static uint_t hash_from_key_fn(addr_t hashkey) {    
-    uint8_t * key = (uint8_t *)hashkey;
-    return v3_hash_buffer(key, HASH_KEY_LEN);
-}
 
-static int hash_key_equal(addr_t key1, addr_t key2) {
-    uint8_t * buf1 = (uint8_t *)key1;
-    uint8_t * buf2 = (uint8_t *)key2;
-    return (memcmp(buf1, buf2, HASH_KEY_LEN) == 0);
-}
 
-static int init_route_cache() {
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, &hash_from_key_fn, &hash_key_equal);
+static struct {
+    struct list_head routes;
+    struct list_head devs;
+    
+    int num_routes;
+    int num_devs;
 
-    if (g_route_cache == NULL){
-        PrintError("Vnet: Route Cache Initiate Failurely\n");
-        return -1;
-    }
+    struct vnet_brg_dev * bridge;
 
-    return 0;
-}
+    v3_lock_t lock;
 
-static void make_hash_key(route_hashkey_t hashkey, char src_addr[6], char dest_addr[6], char src_type, int src_index) {
-    int j;
+    struct hashtable * route_cache;
 
-    for (j = 0; j < 6; j++) {
-       hashkey[j] = src_addr[j];
-       hashkey[j + 6] = dest_addr[j] + 1;
-    }
+    struct bridge_pkts_buf in_buf;  //incoming packets buffer
+} vnet_state;
 
-    hashkey[12] = src_type;
 
-    *(int *)(hashkey + 12) = src_index;
-}
-static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int * matches) {
-    struct route_cache_entry * new_entry = NULL;
-    int i;
-    
-    new_entry = (struct route_cache_entry *)V3_Malloc(sizeof(struct route_cache_entry));
-    if (new_entry == NULL){
-       PrintError("Vnet: Malloc fails\n");
-       return -1;
-    }
-    
-    new_entry->num_matched_routes = num_matched_r;
-
-    new_entry->matches = (int *)V3_Malloc(num_matched_r * sizeof(int));
-    
-    if (new_entry->matches == NULL){
-       PrintError("Vnet: Malloc fails\n");
-       return -1;
-    }
-    
-    for (i = 0; i < num_matched_r; i++) {
-       new_entry->matches[i] = matches[i];
-    }
-    
-    //here, when v3_htable_insert return 0, it means insert fails
-    if (v3_htable_insert(g_route_cache, (addr_t)hashkey, (addr_t)new_entry) == 0){
-       PrintError("Vnet: Insert new route entry to cache failed\n");
-       V3_Free(new_entry->matches);
-       V3_Free(new_entry);
-    }
-    
-    return 0;
-}
 
-static int clear_hash_cache() {
-    v3_free_htable(g_route_cache, 1, 1);
-               
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, hash_from_key_fn, hash_key_equal);
-    
-    if (g_route_cache == NULL){
-        PrintError("Vnet: Route Cache Create Failurely\n");
-        return -1;
-    }
 
-    return 0;
+#ifdef CONFIG_DEBUG_VNET
+static inline void mac_to_string(uint8_t mac[6], char * buf) {
+    snprintf(buf, 100, "%d:%d:%d:%d:%d:%d", 
+            mac[0], mac[1], mac[2],
+            mac[3], mac[4], mac[5]);
 }
 
-static int look_into_cache(route_hashkey_t hashkey, int * matches) {
-    int n_matches = -1;
-    int i = 0;
-    struct route_cache_entry * found = NULL;
-    
-    found = (struct route_cache_entry *)v3_htable_search(g_route_cache, (addr_t)hashkey);
-   
-    if (found != NULL) {
-        n_matches = found->num_matched_routes;
-
-        for (i = 0; i < n_matches; i++) {
-            matches[i] = found->matches[i];
-       }
-    }
+static void print_route(struct vnet_route_info * route){
+    char str[50];
 
-    return n_matches;
-}
+    memset(str, 0, 50);
 
+    mac_to_string(route->route_def.src_mac, str);
+    PrintDebug("Src Mac (%s),  src_qual (%d)\n", 
+                       str, route->route_def.src_mac_qual);
 
+    mac_to_string(route->route_def.dst_mac, str);
+    PrintDebug("Dst Mac (%s),  dst_qual (%d)\n", 
+                       str, route->route_def.dst_mac_qual);
 
+    PrintDebug("Src dev id (%d), src type (%d)", 
+                       route->route_def.src_id, 
+                       route->route_def.src_type);
 
-static inline uint8_t hex_nybble_to_nybble(const uint8_t hexnybble) {
-    uint8_t x = toupper(hexnybble);
+    PrintDebug("Dst dev id (%d), dst type (%d)\n", 
+                       route->route_def.dst_id, 
+                       route->route_def.dst_type);
 
-    if (isdigit(x)) {
-       return x - '0';
-    } else {
-       return 10 + (x - 'A');
+    if (route->route_def.dst_type == LINK_INTERFACE) {
+       PrintDebug("dst_dev (%p), dst_dev_id (%d), dst_dev_input (%p), dst_dev_data (%p)\n",
+                  route->dst_dev,
+                  route->dst_dev->dev_id,
+                  route->dst_dev->input,
+                  route->dst_dev->private_data);
     }
 }
 
-static inline uint8_t hex_byte_to_byte(const uint8_t hexbyte[2]) {
-    return ((hex_nybble_to_nybble(hexbyte[0]) << 4) + 
-           (hex_nybble_to_nybble(hexbyte[1]) & 0xf));
-}
+static void dump_routes() {
+       struct vnet_route_info * route = NULL;
+       int i = 0;
 
+       PrintDebug("\n========Dump routes starts ============\n");
 
-static inline void string_to_mac(const char * str, uint8_t mac[6]) {
-    int k;
-
-    for (k = 0; k < 6; k++) {
-       mac[k] = hex_byte_to_byte(&(str[(2 * k) + k]));
-    }
-}
+       list_for_each_entry(route, &(vnet_state.routes), node) {
+               PrintDebug("\nroute %d:\n", i++);
+               print_route(route);
+       }
 
-static inline void mac_to_string(int mac[6], char * buf) {
-    snprintf(buf, 20, "%x:%x:%x:%x:%x:%x", 
-            mac[0], mac[1], mac[2],
-            mac[3], mac[4], mac[5]);
+       PrintDebug("\n========Dump routes end ============\n");
 }
 
-#if 0
-static void ip_to_string(uint32_t addr, char * buf) {
-    uint32_t addr_st;
-    char * tmp_str;
-    
-    addr_st = v3_htonl(addr);
-    tmp_str = v3_inet_ntoa(addr_st);
-    
-    memcpy(buf, tmp_str, strlen(tmp_str));
-}
 #endif
 
-int find_link_by_fd(SOCK sock) {
-    int i;
 
-    FOREACH_LINK(i, g_links, g_first_link) {
-       if (g_links[i].link_sock == sock) {
-           return i;
-       }
-    }
-    
-    return -1;
+/* 
+ * A VNET packet is a packed struct with the hashed fields grouped together.
+ * This means we can generate the hash from an offset into the pkt struct
+ */
+static inline uint_t hash_fn(addr_t hdr_ptr) {    
+    uint8_t * hdr_buf = (uint8_t *)hdr_ptr;
+
+    return v3_hash_buffer(hdr_buf, VNET_HASH_SIZE);
 }
 
-int vnet_add_link_entry(unsigned long dest, int type, int data_port,  SOCK fd) {
-    int i;
-    
-    for (i = 0; i < MAX_LINKS; i++) {
-       if (g_links[i].use == 0) {
-           g_links[i].dest = dest;
-           g_links[i].type = type;
-           g_links[i].link_sock = fd;
-           g_links[i].remote_port = data_port;
-           g_links[i].use = 1;
-
-           if (g_first_link == -1) {
-               g_first_link = i;
-           }
+static inline int hash_eq(addr_t key1, addr_t key2) {  
+    return (memcmp((uint8_t *)key1, (uint8_t *)key2, VNET_HASH_SIZE) == 0);
+}
 
-           g_links[i].prev = g_last_link;
-           g_links[i].next = -1;
-           
-           if (g_last_link != -1) {
-               g_links[g_last_link].next = i;
-           }
-           
-           g_last_link = i;
-           
-           g_num_links++;
-           
-           return i;
-       }
+static int add_route_to_cache(const struct v3_vnet_pkt * pkt, struct route_list * routes) {
+    memcpy(routes->hash_buf, pkt->hash_buf, VNET_HASH_SIZE);    
+
+    if (v3_htable_insert(vnet_state.route_cache, (addr_t)routes->hash_buf, (addr_t)routes) == 0) {
+       PrintError("Vnet: Failed to insert new route entry to the cache\n");
+       return -1;
     }
     
-    return -1;
+    return 0;
 }
 
+static int clear_hash_cache() {
 
-int add_sock(struct sock_list *socks, int  len, int *first_sock, int *last_sock, SOCK fd) {
-    int i;
+    v3_free_htable(vnet_state.route_cache, 1, 1);
+    vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
 
-    for (i = 0; i < len; i++) {
-       if (socks[i].sock == -1) {
-           socks[i].sock = fd;
-           
-           if (*first_sock == -1) {
-               *first_sock = i;
-           }
-           
-           socks[i].prev = *last_sock;
-           socks[i].next = -1;
-           
-           if (*last_sock != -1) {
-               socks[*last_sock].next = i;
-           }
-           
-           *last_sock = i;
-           
-           return i;
-       }
-    }
-    return -1;
+    return 0;
 }
 
-
-int vnet_add_route_entry(char src_mac[6], char dest_mac[6], int src_mac_qual, int dest_mac_qual, int dest, int type, int src, int src_type) {
-    int i;
-    
-    for(i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes[i].use == 0) {
-           
-           if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-               memcpy(g_routes[i].src_mac, src_mac, 6);
-           } else {
-               memset(g_routes[i].src_mac, 0, 6);
-           }
-           
-           if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-               memcpy(g_routes[i].dest_mac, dest_mac, 6);
-           } else {
-               memset(g_routes[i].dest_mac, 0, 6);
-           }
-           
-           g_routes[i].src_mac_qual = src_mac_qual;
-           g_routes[i].dest_mac_qual = dest_mac_qual;
-           g_routes[i].dest = dest;
-           g_routes[i].type = type;
-           g_routes[i].src = src;
-           g_routes[i].src_type = src_type;
-           g_routes[i].use = 1;
-           
-           if (g_first_route == -1) {
-               g_first_route = i;
-           }
-           
-           g_routes[i].prev = g_last_route;
-           g_routes[i].next = -1;
-           
-           if (g_last_route != -1) {
-               g_routes[g_last_route].next = i;
-           }
-           
-           g_last_route = i;
-           
-           g_num_routes++;
-           
-           return i;
-       }
-    }
+static int look_into_cache(const struct v3_vnet_pkt * pkt, struct route_list ** routes) {
     
-    clear_hash_cache();
-    
-    return -1;
+    *routes = (struct route_list *)v3_htable_search(vnet_state.route_cache, (addr_t)(pkt->hash_buf));
+   
+    return 0;
 }
 
 
-static int find_link_entry(unsigned long dest, int type) {
-    int i;
-
-    FOREACH_LINK(i, g_links, g_first_link) {
-       if ((g_links[i].dest == dest) && 
-           ((type == -1) || (g_links[i].type == type)) ) {
-           return i;
-       }
-    } 
+static struct vnet_dev * find_dev_by_id(int idx) {
+    struct vnet_dev * dev = NULL; 
     
-    return -1;
-}
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       int dev_id = dev->dev_id;
 
-static int delete_link_entry(int index) {
-    int next_i;
-    int prev_i;
-  
-    if (g_links[index].use == 0) {
-       return -1;
+       if (dev_id == idx) {
+           return dev;
+       }
     }
 
-    g_links[index].dest = 0;
-    g_links[index].type = 0;
-    g_links[index].link_sock = -1;
-    g_links[index].use = 0;
-
-    prev_i = g_links[index].prev;
-    next_i = g_links[index].next;
-
-    if (prev_i != -1) {
-       g_links[prev_i].next = g_links[index].next;
-    }    
+    return NULL;
+}
 
-    if (next_i != -1) {
-       g_links[next_i].prev = g_links[index].prev;
-    }
+static struct vnet_dev * find_dev_by_mac(char mac[6]) {
+    struct vnet_dev * dev = NULL; 
     
-    if (g_first_link == index) {
-       g_first_link = g_links[index].next;
-    }    
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       if (memcmp(dev->mac_addr, mac, 6) == 0) {
+           return dev;
+       }
+    }
 
-    if (g_last_link == index) {
-       g_last_link = g_links[index].prev;
-    }    
+    return NULL;
+}
 
-    g_links[index].next = -1;
-    g_links[index].prev = -1;
+int get_device_id_by_mac(char mac[6]) {
+    struct vnet_dev * dev = find_dev_by_mac(mac);
     
-    g_num_links--;
+    if (dev == NULL) {
+       return -1;
+    }
     
-    return 0;
+    return dev->dev_id;
 }
 
-int vnet_delete_link_entry_by_addr(unsigned long dest, int type) {
-    int index = find_link_entry(dest, type);
-  
-    if (index == -1) {
-       return -1;
-    }
 
-    return delete_link_entry(index);
-}
+int v3_vnet_add_route(struct v3_vnet_route route) {
+    struct vnet_route_info * new_route = NULL;
+    uint32_t flags = 0; 
 
+    new_route = (struct vnet_route_info *)V3_Malloc(sizeof(struct vnet_route_info));
+    memset(new_route, 0, sizeof(struct vnet_route_info));
 
-static int find_route_entry(char src_mac[6], 
-                           char dest_mac[6], 
-                           int src_mac_qual, 
-                           int dest_mac_qual, 
-                           int dest, 
-                           int type, 
-                           int src, 
-                           int src_type) {
-    int i;
-    char temp_src_mac[6];
-    char temp_dest_mac[6];
-  
-    if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-       memcpy(temp_src_mac, src_mac, 6);
-    } else {
-       memset(temp_src_mac, 0, 6);
-    }
-    
-    if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-       memcpy(temp_dest_mac, dest_mac, 6);
-    } else {
-       memset(temp_dest_mac, 0, 6);
-    }
+    PrintDebug("Vnet: vnet_add_route_entry: dst_id: %d, dst_type: %d\n",
+                       route.dst_id, route.dst_type);  
     
-    FOREACH_LINK(i, g_routes, g_first_route) {
-       if ( (memcmp(temp_src_mac, g_routes[i].src_mac, 6) == 0) && 
-            (memcmp(temp_dest_mac, g_routes[i].dest_mac, 6) == 0) &&
-            (g_routes[i].src_mac_qual == src_mac_qual) &&
-            (g_routes[i].dest_mac_qual == dest_mac_qual)  &&
-            ( (type == -1) || 
-              ( (type == g_routes[i].type) && (g_routes[i].dest == dest)) ) &&
-            ( (src_type == -1) || 
-              ( (src_type == g_routes[i].src_type) && (g_routes[i].src == src)) ) ) {
-           return i;
-       }
-    } 
-    
-    return -1;
-}
+    memcpy(new_route->route_def.src_mac, route.src_mac, 6);
+    memcpy(new_route->route_def.dst_mac, route.dst_mac, 6);
+    new_route->route_def.src_mac_qual = route.src_mac_qual;
+    new_route->route_def.dst_mac_qual = route.dst_mac_qual;
+    new_route->route_def.dst_id = route.dst_id;
+    new_route->route_def.dst_type = route.dst_type;
+    new_route->route_def.src_id = route.src_id;
+    new_route->route_def.src_type = route.src_type;
 
-static int delete_route_entry(int index) {
-    int next_i;
-    int prev_i;
-
-    memset(g_routes[index].src_mac, 0, 6);
-    memset(g_routes[index].dest_mac, 0, 6);
-
-    g_routes[index].dest = 0;
-    g_routes[index].src = 0;
-    g_routes[index].src_mac_qual = 0;
-    g_routes[index].dest_mac_qual = 0;
-    g_routes[index].type = -1;
-    g_routes[index].src_type = -1;
-    g_routes[index].use = 0;
-
-    prev_i = g_routes[index].prev;
-    next_i = g_routes[index].next;
-  
-    if (prev_i != -1) {
-       g_routes[prev_i].next = g_routes[index].next;
+    if (new_route->route_def.dst_type == LINK_INTERFACE) {
+       new_route->dst_dev = find_dev_by_id(new_route->route_def.dst_id);
+       PrintDebug("Vnet: Add route, get device: dev_id %d, input : %p, private_data %p\n",
+                       new_route->dst_dev->dev_id, new_route->dst_dev->input, new_route->dst_dev->private_data);
     }
-    
-    if (next_i != -1) {
-       g_routes[next_i].prev = g_routes[index].prev;
+
+    if (new_route->route_def.src_type == LINK_INTERFACE) {
+       new_route->src_dev = find_dev_by_id(new_route->route_def.src_id);
     }
-    
-    if (g_first_route == index) {
-       g_first_route = g_routes[index].next;
-    }    
 
-    if (g_last_route == index) {
-       g_last_route = g_routes[index].prev;
-    }    
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    g_routes[index].next = -1;
-    g_routes[index].prev = -1;
-    
-    g_num_routes--;
-    
+    list_add(&(new_route->node), &(vnet_state.routes));
     clear_hash_cache();
-    
-    return 0;
-}
 
-int vnet_delete_route_entry_by_addr(char src_mac[6], 
-                                   char dest_mac[6], 
-                                   int src_mac_qual, 
-                                   int dest_mac_qual, 
-                                   int dest, 
-                                   int type, 
-                                   int src, 
-                                   int src_type) {
-    int index = find_route_entry(src_mac, dest_mac, src_mac_qual, 
-                                dest_mac_qual, dest, type, src, src_type);
-    
-    if (index == -1) {
-       return -1;
-    }
-    
-    delete_route_entry(index);
-    
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+   
+
+#ifdef CONFIG_DEBUG_VNET
+    dump_routes();
+#endif
+
     return 0;
 }
 
-int delete_sock(struct sock_list * socks, int * first_sock, int * last_sock, SOCK fd) {
-    int i;
-    int prev_i;
-    int next_i;
-    
-  
-    FOREACH_SOCK(i, socks, (*first_sock)) {
-       if (socks[i].sock == fd) {
-           V3_Close_Socket(socks[i].sock);
-           socks[i].sock = -1;
-           
-           prev_i = socks[i].prev;
-           next_i = socks[i].next;
-           
-           if (prev_i != -1) {
-               socks[prev_i].next = socks[i].next;
-           }
-           
-           if (next_i != -1) {
-               socks[next_i].prev = socks[i].prev;
-           }
-           
-           if (*first_sock == i) {
-               *first_sock = socks[i].next;
-           }
-           
-           if (*last_sock == i) {
-               *last_sock = socks[i].prev;
-           }
-           
-           socks[i].next = -1;
-           socks[i].prev = -1;
-           
-           return 0;
-       }
-    }
-    return -1;
-}
 
-//setup the topology of the testing network
-static void store_topologies(SOCK fd) {
-    int i;
-    int src_mac_qual = MAC_ANY;
-    int dest_mac_qual = MAC_ANY;
-    uint_t dest;
-#ifndef VNET_SERVER
-    dest = (0 | 172 << 24 | 23 << 16 | 1 );
-    PrintDebug("VNET: store_topologies. NOT VNET_SERVER, dest = %x\n", dest);
-#else
-    dest = (0 | 172 << 24 | 23 << 16 | 2 );
-    PrintDebug("VNET: store_topologies. VNET_SERVER, dest = %x\n", dest);
-#endif
 
-    int type = UDP_TYPE;
-    int src = 0;
-    int src_type= LINK_ANY; //ANY_SRC_TYPE
-    int data_port = 22;
-    
-    //store link table
-    for (i = 0; i < MAX_LINKS; i++) {
-       if (g_links[i].use == 0) {
-           g_links[i].dest = (int)dest;
-           g_links[i].type = type;
-           g_links[i].link_sock = fd;
-           g_links[i].remote_port = data_port;
-           g_links[i].use = 1;
-           
-           if (g_first_link == -1) {
-               g_first_link = i;
-           }       
+// At the end allocate a route_list
+// This list will be inserted into the cache so we don't need to free it
+static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
+    struct vnet_route_info * route = NULL; 
+    struct route_list * matches = NULL;
+    int num_matches = 0;
+    int max_rank = 0;
+    struct list_head match_list;
+    struct eth_hdr * hdr = (struct eth_hdr *)(pkt->data);
+    uint8_t src_type = pkt->src_type;
+    uint32_t src_link = pkt->src_id;
 
-           g_links[i].prev = g_last_link;
-           g_links[i].next = -1;
-           
-           if (g_last_link != -1) {
-               g_links[g_last_link].next = i;
-           }
-           
-           g_last_link = i;
-           
-           g_num_links++;
-           PrintDebug("VNET: store_topologies. new link: socket: %d, remote %x:[%d]\n", g_links[i].link_sock, (uint_t)g_links[i].dest, g_links[i].remote_port);
-       }
-    }
-    
-    
-    //store route table
-    
-    type = LINK_EDGE;
-    dest = 0;
-    
-    for (i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes[i].use == 0) {
-           if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-               //                  memcpy(g_routes[i].src_mac, src_mac, 6);
-           } else {
-               memset(g_routes[i].src_mac, 0, 6);
-           }
-           
-           if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-               //                  memcpy(g_routes[i].dest_mac, dest_mac, 6);
-           } else {
-               memset(g_routes[i].dest_mac, 0, 6);
-           }
-           
-           g_routes[i].src_mac_qual = src_mac_qual;
-           g_routes[i].dest_mac_qual = dest_mac_qual;
-           g_routes[i].dest = (int)dest;
-           g_routes[i].type = type;
-           g_routes[i].src = src;
-           g_routes[i].src_type = src_type;
-           
-           g_routes[i].use = 1;
-           
-           if (g_first_route == -1) {
-               g_first_route = i;
-           }
-           
-           g_routes[i].prev = g_last_route;
-           g_routes[i].next = -1;
-           
-           if (g_last_route != -1) {
-               g_routes[g_last_route].next = i;
-           }
-           
-           g_last_route = i;
-           
-           g_num_routes++;
-           
-           PrintDebug("VNET: store_topologies. new route: src_mac: %s, dest_mac: %s, dest: %d\n", g_routes[i].src_mac, g_routes[i].dest_mac, dest);
-           
-       }
+#ifdef CONFIG_DEBUG_VNET
+    {
+       char dst_str[100];
+       char src_str[100];
+
+       mac_to_string(hdr->src_mac, src_str);  
+       mac_to_string(hdr->dst_mac, dst_str);
+       PrintDebug("Vnet: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
     }
-}
+#endif
 
-static int match_route(uint8_t * src_mac, uint8_t * dst_mac, int src_type, int src_index, int * matches) { 
-    int values[MAX_ROUTES];
-    int matched_routes[MAX_ROUTES];
-    
-    int num_matches = 0;
-    int i;
-    int max = 0;
-    int no = 0;
-    int exact_match = 0;
-    
-    FOREACH_ROUTE(i, g_routes, g_first_route) {
-       if ((g_routes[i].src_type != LINK_ANY) &&
-           ((g_routes[i].src_type != src_type) ||
-            ((g_routes[i].src != src_index) &&
-             (g_routes[i].src != -1)))) {
-           PrintDebug("Vnet: MatchRoute: Source route is on and does not match\n");
+    INIT_LIST_HEAD(&match_list);
+    
+#define UPDATE_MATCHES(rank) do {                              \
+       if (max_rank < (rank)) {                                \
+           max_rank = (rank);                                  \
+           INIT_LIST_HEAD(&match_list);                        \
+                                                               \
+           list_add(&(route->match_node), &match_list);        \
+           num_matches = 1;                                    \
+       } else if (max_rank == (rank)) {                        \
+           list_add(&(route->match_node), &match_list);        \
+           num_matches++;                                      \
+       }                                                       \
+    } while (0)
+    
+
+    list_for_each_entry(route, &(vnet_state.routes), node) {
+       struct v3_vnet_route * route_def = &(route->route_def);
+
+       // CHECK SOURCE TYPE HERE
+       if ( (route_def->src_type != LINK_ANY) && 
+            ( (route_def->src_type != src_type) || 
+              ( (route_def->src_id != src_link) &&
+                (route_def->src_id != (uint32_t)-1)))) {
            continue;
        }
-       
-       if ( (g_routes[i].dest_mac_qual == MAC_ANY) &&
-            (g_routes[i].src_mac_qual == MAC_ANY) ) {      
-           matched_routes[num_matches] = i;
-           values[num_matches] = 3;
-           num_matches++;
+
+
+       if ((route_def->dst_mac_qual == MAC_ANY) &&
+           (route_def->src_mac_qual == MAC_ANY)) {      
+           UPDATE_MATCHES(3);
        }
        
-       if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-           if (g_routes[i].src_mac_qual !=  MAC_NOT) {
-               if (g_routes[i].dest_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 6;
-                   
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-                   if (g_routes[i].dest_mac_qual != MAC_NOT) {   
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 8;    
-                       exact_match = 1;
-                       num_matches++;
-                   }
+       if (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) {
+           if (route_def->src_mac_qual != MAC_NOT) {
+               if (route_def->dst_mac_qual == MAC_ANY) {
+                   UPDATE_MATCHES(6);
+               } else if (route_def->dst_mac_qual != MAC_NOT &&
+                          memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
+                   UPDATE_MATCHES(8);
                }
            }
        }
-       
-       if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-           if (g_routes[i].dest_mac_qual != MAC_NOT) {
-               if (g_routes[i].src_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 6;
-                   
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-                   if (g_routes[i].src_mac_qual != MAC_NOT) {
-                       if (exact_match == 0) {
-                           matched_routes[num_matches] = i;
-                           values[num_matches] = 8;
-                           num_matches++;
-                       }
-                   }
+           
+       if (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
+           if (route_def->dst_mac_qual != MAC_NOT) {
+               if (route_def->src_mac_qual == MAC_ANY) {
+                   UPDATE_MATCHES(6);
+               } else if ((route_def->src_mac_qual != MAC_NOT) && 
+                          (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {
+                   UPDATE_MATCHES(8);
                }
            }
        }
-       
-       if ((g_routes[i].dest_mac_qual == MAC_NOT) &&
-           (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) != 0)) {
-           if (g_routes[i].src_mac_qual == MAC_ANY) {
-               matched_routes[num_matches] = i;
-               values[num_matches] = 5;                    
-               num_matches++;    
-           } else if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-               if (g_routes[i].src_mac_qual != MAC_NOT) {      
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 7;                  
-                   num_matches++;
-               }
+           
+       if ((route_def->dst_mac_qual == MAC_NOT) &&
+           (memcmp(route_def->dst_mac, hdr->dst_mac, 6) != 0)) {
+           if (route_def->src_mac_qual == MAC_ANY) {
+               UPDATE_MATCHES(5);
+           } else if ((route_def->src_mac_qual != MAC_NOT) && 
+                      (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {     
+               UPDATE_MATCHES(7);
            }
        }
        
-       if ((g_routes[i].src_mac_qual == MAC_NOT) &&
-           (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) != 0)) {
-           if (g_routes[i].dest_mac_qual == MAC_ANY) {
-               matched_routes[num_matches] = i;
-               values[num_matches] = 5;            
-               num_matches++;
-           } else if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-               if (g_routes[i].dest_mac_qual != MAC_NOT) { 
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 7;
-                   num_matches++;
-               }
+       if ((route_def->src_mac_qual == MAC_NOT) &&
+           (memcmp(route_def->src_mac, hdr->src_mac, 6) != 0)) {
+           if (route_def->dst_mac_qual == MAC_ANY) {
+               UPDATE_MATCHES(5);
+           } else if ((route_def->dst_mac_qual != MAC_NOT) &&
+                      (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0)) {
+               UPDATE_MATCHES(7);
            }
        }
+       
+       // Default route
+       if ( (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) &&
+            (route_def->dst_mac_qual == MAC_NONE)) {
+           UPDATE_MATCHES(4);
+       }
     }
-    //end FOREACH_ROUTE
-    
-    FOREACH_ROUTE(i, g_routes, g_first_route) {
-       if ((memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) &&
-           (g_routes[i].dest_mac_qual == MAC_NONE) &&
-           ((g_routes[i].src_type == LINK_ANY) ||
-            ((g_routes[i].src_type == src_type) &&
-             ((g_routes[i].src == src_index) ||
-              (g_routes[i].src == -1))))) {
-           matched_routes[num_matches] = i;
-           values[num_matches] = 4;
-           PrintDebug("Vnet: MatchRoute: We matched a default route (%d)\n", i);
-           num_matches++;
-       }
-    }
-    
-    //If many rules have been matched, we choose one which has the highest value rating
-    if (num_matches == 0) {
-       return 0;
-    }
-    
-    for (i = 0; i < num_matches; i++) {
-       if (values[i] > max) {
-           no = 0;
-           max = values[i];
-           matches[no] = matched_routes[i];
-           no++;
-       } else if (values[i] == max) {
-           matches[no] = matched_routes[i];
-           no++;
-       }
-    }
-    
-    return no;
-}
 
+    PrintDebug("Vnet: match_route: Matches=%d\n", num_matches);
 
-static int process_udpdata() {
-    struct raw_ethernet_pkt * pt;
+    if (num_matches == 0) {
+       return NULL;
+    }
 
-    uint32_t dest = 0;
-    uint16_t remote_port = 0;
-    SOCK link_sock = g_udp_sockfd;
-    int length = sizeof(struct raw_ethernet_pkt) - (2 * sizeof(int));   //minus the "size" and "type" 
+    matches = (struct route_list *)V3_Malloc(sizeof(struct route_list) + 
+                               (sizeof(struct vnet_route_info *) * num_matches));
 
-    //run in a loop to get packets from outside network, adding them to the incoming packet queue
-    while (1) {
-       pt = (struct raw_ethernet_pkt *)V3_Malloc(sizeof(struct raw_ethernet_pkt));
+    matches->num_routes = num_matches;
 
-       if (pt == NULL){
-           PrintError("Vnet: process_udp: Malloc fails\n");
-           continue;
-       }
-       
-       PrintDebug("Vnet: route_thread: socket: [%d]. ready to receive from ip [%x], port [%d] or from VMs\n", link_sock, (uint_t)dest, remote_port);
-       pt->size = V3_RecvFrom_IP( link_sock, dest, remote_port, pt->data, length);
-       PrintDebug("Vnet: route_thread: socket: [%d] receive from ip [%x], port [%d]\n", link_sock, (uint_t)dest, remote_port);
-       
-       if (pt->size <= 0) {
-           PrintDebug("Vnet: process_udp: receiving packet from UDP fails\n");
-           V3_Free(pt);
-           return -1;
+    {
+       int i = 0;
+       list_for_each_entry(route, &match_list, match_node) {
+           matches->routes[i++] = route;
        }
-       
-       PrintDebug("Vnet: process_udp: get packet\n");
-       print_packet(pt->data, pt->size);
+    }
+
+    return matches;
+}
 
+#if 0
+static int flush_bridge_pkts(struct vnet_brg_dev *bridge){
+    uint32_t flags;
+    int num;
+    int start;
+    int send;
+    struct v3_vnet_bridge_input_args args;
+    int cpu_id = bridge->vm->cores[0].cpu_id;
+    int current_core = V3_Get_CPU();
        
-       //V3_Yield();
+    if (bridge == NULL) {
+       PrintDebug("VNET: No bridge to sent data to links\n");
+       return -1;
     }
-}
 
+    flags = v3_lock_irqsave(bridge->recv_buf.lock);
+               
+    num = bridge->recv_buf.num;
+    start = bridge->recv_buf.start;
 
+    bridge->recv_buf.num -= num;
+    bridge->recv_buf.start += num;
+    bridge->recv_buf.start %= BRIDGE_BUF_SIZE;
+       
+    v3_unlock_irqrestore(bridge->recv_buf.lock, flags);
 
-static int indata_handler( )
-{
-      if (!use_tcp)
-          process_udpdata();     
 
-      return 0;   
-}
+    if (bridge->disabled) {
+       PrintDebug("VNET: In flush bridge pkts: Bridge is disabled\n");
+       return -1;
+    }
 
-static int start_recv_data()
-{
-       if (use_tcp){
-               
+    if (num <= 2 && num > 0) {
+       PrintDebug("VNET: In flush bridge pkts: %d\n", num);
+    }
+
+    if (num > 0) {
+       PrintDebug("VNET: In flush bridge pkts to bridge, cur_cpu %d, brige_core: %d\n", current_core, cpu_id);
+       if (current_core == cpu_id) {
+           if ((start + num) < BRIDGE_BUF_SIZE) { 
+               bridge->input(bridge->vm, &(bridge->recv_buf.pkts[start]), num, bridge->private_data);
+           } else {
+               bridge->input(bridge->vm, &(bridge->recv_buf.pkts[start]), (BRIDGE_BUF_SIZE - start), bridge->private_data);                            
+               send = num - (BRIDGE_BUF_SIZE - start);
+               bridge->input(bridge->vm, &(bridge->recv_buf.pkts[0]), send, bridge->private_data);
+           }   
        } else {
-               SOCK udp_data_socket;
-  
-               if ((udp_data_socket = V3_Create_UDP_Socket()) < 0){
-                       PrintError("VNET: Can't setup udp socket\n");
-                       return -1; 
-               }
-               PrintDebug("Vnet: vnet_setup_udp: get socket: %d\n", udp_data_socket);
-               g_udp_sockfd = udp_data_socket;
-
-               store_topologies(udp_data_socket);
-
-               if (V3_Bind_Socket(udp_data_socket, vnet_udp_port) < 0){ 
-                       PrintError("VNET: Can't bind socket\n");
-                       return -1;
-               }
-               PrintDebug("VNET: vnet_setup_udp: bind socket successful\n");
+           args.vm = bridge->vm;
+           args.private_data = bridge->private_data;
+       
+           if ((start + num) < BRIDGE_BUF_SIZE) {
+               args.pkt_num = num;
+               args.vnet_pkts = &(bridge->recv_buf.pkts[start]);
+               V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
+           } else {
+               args.pkt_num = BRIDGE_BUF_SIZE - start;
+               args.vnet_pkts = &(bridge->recv_buf.pkts[start]);
+               V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
+                               
+               send = num - (BRIDGE_BUF_SIZE - start);
+               args.pkt_num = send;
+               args.vnet_pkts = &(bridge->recv_buf.pkts[0]);                   
+               V3_Call_On_CPU(cpu_id, bridge->xcall_input, (void *)&args);
+           }
        }
-
-       V3_CREATE_THREAD(&indata_handler, NULL, "VNET_DATA_HANDLER");
-       return 0;
+       
+       PrintDebug("VNET: flush bridge pkts %d\n", num);
+    }
+                       
+    return 0;
 }
+#endif
 
+static int send_to_bridge(struct v3_vnet_pkt * pkt){
+    struct vnet_brg_dev * bridge = vnet_state.bridge;
 
-static inline int if_write_pkt(struct vnet_if_device *iface, struct raw_ethernet_pkt * pkt) {
-    return iface->input((uchar_t *)pkt->data, pkt->size);
-}
+    if (bridge == NULL) {
+       PrintDebug("VNET: No bridge to sent data to links\n");
+       return -1;
+    }
 
-static int handle_one_pkt(struct raw_ethernet_pkt * pkt) {
-    int src_link_index = 0;    //the value of src_link_index of udp always is 0
-    int i;
-    char src_mac[6];
-    char dst_mac[6];
+    if (bridge->max_delayed_pkts <= 1) {
 
-    int matches[g_num_routes];
-    int num_matched_routes = 0;
+       if (bridge->disabled) {
+           PrintDebug("VNET: Bridge diabled\n");
+           return -1;
+       }
 
-    struct HEADERS headers;
-  
-    // get the ethernet and ip headers from the packet
-    memcpy((void *)&headers, (void *)pkt->data, sizeof(headers));
+       bridge->input(bridge->vm, pkt, 1, bridge->private_data);
 
-    int j;
-    for (j = 0;j < 6; j++) {
-       src_mac[j] = headers.ethernetsrc[j];
-       dst_mac[j] = headers.ethernetdest[j];
+       PrintDebug("VNET: sent one packet to the bridge\n");
     }
 
 
-#ifdef DEBUG
-    char dest_str[18];
-    char src_str[18];
-    
-    mac_to_string(src_mac, src_str);  
-    mac_to_string(dst_mac, dest_str);
-    
-    PrintDebug("Vnet: HandleDataOverLink. SRC(%s), DEST(%s)\n", src_str, dest_str);
-#endif
-    
-    char hash_key[hash_key_size];
-    make_hash_key(hash_key, src_mac, dst_mac, LINK_EDGE, src_link_index);//link_edge -> pt->type???
-    
-    num_matched_routes = look_into_cache((route_hashkey_t)hash_key, matches);
+    return 0;
+}
+
+int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
+    struct route_list * matched_routes = NULL;
+    uint32_t flags = 0;
+    int i = 0;
     
-    if (num_matched_routes == -1) {  //no match
-        num_matched_routes = match_route(src_mac, dst_mac, pkt->type, src_link_index, matches);
+#ifdef CONFIG_DEBUG_VNET
+    {
+       struct eth_hdr * hdr = (struct eth_hdr *)(pkt->header);
+       char dest_str[100];
+       char src_str[100];
+       int cpu = V3_Get_CPU();
        
-       if (num_matched_routes > 0) {
-           add_route_to_cache(hash_key, num_matched_routes,matches);      
-       }
-    }
-    
-    PrintDebug("Vnet: HandleDataOverLink: Matches=%d\n", num_matched_routes);
-    
-    for (i = 0; i < num_matched_routes; i++) {
-        int route_index = -1;
-        int link_index = -1;
-        int dev_index = -1;
+       mac_to_string(hdr->src_mac, src_str);  
+       mac_to_string(hdr->dst_mac, dest_str);
+       PrintDebug("Vnet: on cpu %d, HandleDataOverLink. SRC(%s), DEST(%s), pkt size: %d\n", cpu, src_str, dest_str, pkt->size);
+   }
+#endif
+
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    look_into_cache(pkt, &matched_routes);
        
-        route_index = matches[i];
+    if (matched_routes == NULL) {  
+       PrintDebug("Vnet: send pkt Looking into routing table\n");
        
-        PrintDebug("Vnet: HandleDataOverLink: Forward packet from link according to Route entry %d\n", route_index);
+       matched_routes = match_route(pkt);
        
-        if (g_routes[route_index].type == LINK_EDGE) {
-            link_index = g_routes[route_index].dest;
-           
-            if(g_links[link_index].type == UDP_TYPE) {
-                int size;
+       if (matched_routes) {
+           add_route_to_cache(pkt, matched_routes);
+       } else {
+           PrintDebug("Could not find route for packet... discards packet\n");
+           v3_unlock_irqrestore(vnet_state.lock, flags);
+           return -1;
+       }
+    }
 
-                               
-                 PrintDebug("===Vnet: HandleDataOverLink: Serializing UDP Packet to link_sock [%d], dest [%x], remote_port [%d], size [%d]\n", g_links[link_index].link_sock, (uint_t)g_links[link_index].dest,  g_links[link_index].remote_port, (int)pkt->size);
-               
-                if ((size = V3_SendTo_IP(g_links[link_index].link_sock,  g_links[link_index].dest,  g_links[link_index].remote_port, pkt->data, pkt->size)) != pkt->size)  {
-                    PrintError("Vnet: sending by UDP Exception, %x\n", size);
-                    return -1;
-                }
-               
-                PrintDebug("Vnet: HandleDataOverLink: Serializing UDP Packet to link_sock [%d], dest [%x], remote_port [%d], size [%d]\n", g_links[link_index].link_sock, (uint_t)g_links[link_index].dest,  g_links[link_index].remote_port, (int)pkt->size);
-               
-            } else if (g_links[link_index].type == TCP_TYPE) {
-               
-            }
-        } else if (g_routes[route_index].type == LINK_INTERFACE) {
-            dev_index = g_routes[route_index].dest;
-      
-            PrintDebug("Writing Packet to device=%s\n", g_devices[dev_index].device->name);
-
-            if (if_write_pkt(g_devices[dev_index].device, pkt) == -1) {
-               PrintDebug("Can't write output packet to link\n");
-                return -1;
-            }
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+
+    PrintDebug("Vnet: send pkt route matches %d\n", matched_routes->num_routes);
+
+    for (i = 0; i < matched_routes->num_routes; i++) {
+        struct vnet_route_info * route = matched_routes->routes[i];
+       
+        if (route->route_def.dst_type == LINK_EDGE) {                  
+            pkt->dst_type = LINK_EDGE;
+            pkt->dst_id = route->route_def.dst_id;
+
+            if (send_to_bridge(pkt) == -1) {
+                PrintDebug("VNET: Packet not sent properly to bridge\n");
+                continue;
+            }
+            
+        } else if (route->route_def.dst_type == LINK_INTERFACE) {
+            if (route->dst_dev->input(route->dst_dev->vm, pkt, route->dst_dev->private_data) == -1) {
+                PrintDebug("VNET: Packet not sent properly\n");
+                continue;
+            }
         } else {
             PrintDebug("Vnet: Wrong Edge type\n");
+            continue;
         }
+
+        PrintDebug("Vnet: v3_vnet_send_pkt: Forward packet according to Route %d\n", i);
     }
+    
+    return 0;
+}
 
-     return 0;
+void v3_vnet_send_pkt_xcall(void * data) {
+    struct v3_vnet_pkt * pkt = (struct v3_vnet_pkt *)data;
+    v3_vnet_send_pkt(pkt, NULL);
 }
 
-static int send_ethernet_pkt(char * buf, int length) {
-       struct raw_ethernet_pkt * pt;
 
-       pt = (struct raw_ethernet_pkt *)V3_Malloc(sizeof(struct raw_ethernet_pkt));
-       raw_ethernet_packet_init(pt, buf, length);  //====here we copy sending data once 
-       
-       PrintDebug("VNET: vm_send_pkt: transmitting packet: (size:%d)\n", (int)pt->size);
-       print_packet((char *)buf, length);
-       
-       v3_enqueue(g_inpkt_q, (addr_t)pt);
-       return 0;
-       
-}
+void v3_vnet_polling() {
+    uint32_t flags = 0;
+    int num = 0;
+    int start = 0;
+    struct v3_vnet_pkt * buf = NULL;
 
-int v3_Send_pkt(uchar_t *buf, int length) {
-    PrintDebug("VNET: In V3_Send_pkt: pkt length %d\n", length);
-    
-    return send_ethernet_pkt((char *)buf, length);
-}
+    PrintDebug("In vnet pollling: cpu %d\n", V3_Get_CPU());
 
-static int add_device_to_table(struct vnet_if_device*device, int type) {
-    int i;
-    
-    for (i = 0; i < MAX_DEVICES; i++) {
-       if (g_devices[i].use == 0) {
-           g_devices[i].type = type;
-           g_devices[i].use = 1;
-           
-           if (g_first_device == -1) {
-               g_first_device = i;
-           }       
+    flags = v3_lock_irqsave(vnet_state.in_buf.lock);
+               
+    num = vnet_state.in_buf.num;
+    start = vnet_state.in_buf.start;
 
-           g_devices[i].prev = g_last_device;
-           g_devices[i].next = -1;
-           
-           if (g_last_device != -1) {
-               g_devices[g_last_device].next = i;
-           }
+    PrintDebug("VNET: polling pkts %d\n", num);
 
-           g_last_device = i;
-           g_num_devices++;
-           
-           return i;
-       }
-    }
-    
-    return -1;
-}
+    while (num > 0) {
+       buf = &(vnet_state.in_buf.pkts[vnet_state.in_buf.start]);
 
-static int search_device(char *device_name) {
-    int i;
+       v3_vnet_send_pkt(buf, NULL);
 
-    for (i = 0; i < MAX_DEVICES; i++) {
-        if (g_devices[i].use == 1) {
-           if (!strcmp(device_name, g_devices[i].device->name)) {
-               return i;
-           }
-        }
+       vnet_state.in_buf.num--;
+       vnet_state.in_buf.start++;
+       vnet_state.in_buf.start %= BRIDGE_BUF_SIZE;
+       num--;
     }
-    
-    return -1;
+
+    v3_unlock_irqrestore(vnet_state.in_buf.lock, flags);
+
+    return;
 }
 
-static struct vnet_if_device * delete_device_from_table(int index) {
-    int next_i;
-    int prev_i;
-    struct vnet_if_device * device = NULL;
 
-    if (g_devices[index].use == 0) {
-       return NULL;
+int v3_vnet_rx(uint8_t * buf, uint16_t size, uint16_t src_id, uint8_t src_type) {
+    uint32_t flags = 0;
+    int end = 0;
+    struct v3_vnet_pkt * pkt = NULL;
+   
+    flags = v3_lock_irqsave(vnet_state.in_buf.lock);
+           
+    end = vnet_state.in_buf.end;
+    pkt = &(vnet_state.in_buf.pkts[end]);
+
+    if (vnet_state.in_buf.num > BRIDGE_BUF_SIZE){
+       PrintDebug("VNET: bridge rx: buffer full\n");
+       v3_unlock_irqrestore(vnet_state.in_buf.lock, flags);
+       return 0;
     }
 
-    g_devices[index].use = 0;
+    vnet_state.in_buf.num++;
+    vnet_state.in_buf.end++;
+    vnet_state.in_buf.end %= BRIDGE_BUF_SIZE;
 
-    prev_i = g_devices[index].prev;
-    next_i = g_devices[index].next;
+    pkt->size = size;
+    pkt->src_id = src_id;
+    pkt->src_type = src_type;
+    memcpy(pkt->header, buf, ETHERNET_HEADER_LEN);
+    memcpy(pkt->data, buf, size);
 
-    if (prev_i != -1) {
-        g_devices[prev_i].next = g_devices[index].next;
-    }
+       
+    v3_unlock_irqrestore(vnet_state.in_buf.lock, flags);
 
-    if (next_i != -1) {
-        g_devices[next_i].prev = g_devices[index].prev;
-    }
+    return 0;
+}
+       
+
+int v3_vnet_add_dev(struct v3_vm_info * vm, uint8_t mac[6], 
+                   int (*netif_input)(struct v3_vm_info * vm, struct v3_vnet_pkt * pkt, void * private_data), 
+                   void * priv_data){
+    struct vnet_dev * new_dev = NULL;
+    uint32_t flags = 0;
 
-    if (g_first_device == index) {
-        g_first_device = g_devices[index].next;
+    new_dev = (struct vnet_dev *)V3_Malloc(sizeof(struct vnet_dev)); 
+
+    if (new_dev == NULL) {
+       PrintError("VNET: Malloc fails\n");
+       return -1;
     }
+   
+    memcpy(new_dev->mac_addr, mac, 6);
+    new_dev->input = netif_input;
+    new_dev->private_data = priv_data;
+    new_dev->vm = vm;
+    new_dev->dev_id = 0;       
+
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    if (g_last_device == index) {
-        g_last_device = g_devices[index].prev;
+    if (!find_dev_by_mac(mac)) {
+       list_add(&(new_dev->node), &(vnet_state.devs));
+       new_dev->dev_id = ++vnet_state.num_devs;
     }
 
-    g_devices[index].next = -1;
-    g_devices[index].prev = -1;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    device = g_devices[index].device;
-    g_devices[index].device = NULL;
+    // if the device was found previosly the id should still be 0
+    if (new_dev->dev_id == 0) {
+       PrintError("Device Alrady exists\n");
+       return -1;
+    }
 
-    g_num_devices--;
+    PrintDebug("Vnet: Add Device: dev_id %d, input : %p, private_data %p\n",
+              new_dev->dev_id, new_dev->input, new_dev->private_data);
 
-    return device;
+    return new_dev->dev_id;
 }
 
 
-int vnet_register_device(char * dev_name, int (*netif_input)(uchar_t * pkt, uint_t size), void * data) {
-    struct vnet_if_device * dev;
-    
-    dev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device));
-    
-    if (dev == NULL){
-       PrintError("VNET: Malloc fails\n");
-       return -1;
+void v3_vnet_heartbeat(struct guest_info *core){
+    //static long last_time, cur_time;
+
+    if (vnet_state.bridge == NULL) {
+       return;
     }
-    
-    strncpy(dev->name, dev_name, 50);
-    dev->input = netif_input;
-    dev->data = data;
-    
-    if (add_device_to_table(dev, GENERAL_NIC) == -1) {
-       return -1;
+/*     
+    if(vnet_state.bridge->max_delayed_pkts > 1){
+       if(V3_Get_CPU() != vnet_state.bridge->vm->cores[0].cpu_id){
+           rdtscll(cur_time);
+       }
+
+       if ((cur_time - last_time) >= vnet_state.bridge->max_latency) {
+           last_time = cur_time;
+           flush_bridge_pkts(vnet_state.bridge);
+       }
     }
-    
-    return 0;
+*/
+    vnet_state.bridge->polling_pkt(vnet_state.bridge->vm, vnet_state.bridge->private_data);
 }
 
-int vnet_unregister_device(char * dev_name) {
-    int i;
+int v3_vnet_add_bridge(struct v3_vm_info * vm,
+                      int (*input)(struct v3_vm_info * vm, struct v3_vnet_pkt pkt[], uint16_t pkt_num, void * private_data),
+                      void (*xcall_input)(void * data),
+                      int (*poll_pkt)(struct v3_vm_info * vm, void * private_data),
+                      uint16_t max_delayed_pkts,
+                      long max_latency,
+                      void * priv_data) {
 
-    i = search_device(dev_name);
+    uint32_t flags = 0;
+    int bridge_free = 0;
+    struct vnet_brg_dev * tmp_bridge = NULL;    
     
-    if (i == -1) {
-        return -1;
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    if (vnet_state.bridge == NULL) {
+       bridge_free = 1;
+       vnet_state.bridge = (void *)1;
     }
 
-    struct vnet_if_device * device = delete_device_from_table(i);
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    if (device == NULL) {
+    if (bridge_free == 0) {
+       PrintError("Bridge already set\n");
        return -1;
     }
 
-    V3_Free(device);
+    tmp_bridge = (struct vnet_brg_dev *)V3_Malloc(sizeof(struct vnet_brg_dev));
+
+    if (tmp_bridge == NULL) {
+       PrintError("Malloc Fails\n");
+       vnet_state.bridge = NULL;
+       return -1;
+    }
+    
+    tmp_bridge->vm = vm;
+    tmp_bridge->input = input;
+    tmp_bridge->xcall_input = xcall_input;
+    tmp_bridge->polling_pkt = poll_pkt;
+    tmp_bridge->private_data = priv_data;
+    tmp_bridge->disabled = 0;
+
+/*
+    //initial receving buffer
+    tmp_bridge->recv_buf.start = 0;
+    tmp_bridge->recv_buf.end = 0;
+    tmp_bridge->recv_buf.num = 0;
+    if(v3_lock_init(&(tmp_bridge->recv_buf.lock)) == -1){
+       PrintError("VNET: add bridge, error to initiate recv buf lock\n");
+    }
+    int i;
+    for(i = 0; i<BRIDGE_BUF_SIZE; i++){
+       tmp_bridge->recv_buf.pkts[i].data = &(tmp_bridge->recv_buf.datas[i*ETHERNET_PACKET_LEN]);
+    }
+
+*/
+    
+    tmp_bridge->max_delayed_pkts = (max_delayed_pkts < BRIDGE_BUF_SIZE) ? max_delayed_pkts : BRIDGE_BUF_SIZE;
+    tmp_bridge->max_latency = max_latency;
+       
+    // make this atomic to avoid possible race conditions
+    flags = v3_lock_irqsave(vnet_state.lock);
+    vnet_state.bridge = tmp_bridge;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
     return 0;
 }
 
-int v3_Register_pkt_event(int (*netif_input)(uchar_t * pkt, uint_t size)) {
-    return vnet_register_device("NE2000", netif_input, NULL);
-}
 
-int v3_vnet_pkt_process() {
-    struct raw_ethernet_pkt * pt;
+int v3_vnet_disable_bridge() {
+    uint32_t flags = 0; 
+    
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    PrintDebug("VNET: In vnet_check\n");
-       
-    while ((pt = (struct raw_ethernet_pkt *)v3_dequeue(g_inpkt_q)) != NULL) {
-       PrintDebug("VNET: In vnet_check: pt length %d, pt type %d\n", (int)pt->size, (int)pt->type);
-       v3_hexdump(pt->data, pt->size, NULL, 0);
-       
-       if(handle_one_pkt(pt)) {
-           PrintDebug("VNET: vnet_check: handle one packet!\n");  
-       }
-       
-       V3_Free(pt); //be careful here
+    if (vnet_state.bridge != NULL) {
+       vnet_state.bridge->disabled = 1;
     }
-    
+
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+
     return 0;
 }
 
-static void init_link_table() {
-    int i;
 
-    for (i = 0; i < MAX_LINKS; i++) {
-        g_links[i].use = 0;
-        g_links[i].next = -1;
-        g_links[i].prev = -1;
+int v3_vnet_enable_bridge() {
+    uint32_t flags = 0;
+    
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    if (vnet_state.bridge != NULL) {
+       vnet_state.bridge->disabled = 0;
     }
-       
-    g_first_link = -1;
-    g_last_link = -1;
-    g_num_links = 0;
-}
 
-static void init_device_table() {
-    int i;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    for (i = 0; i < MAX_DEVICES; i++) {
-        g_devices[i].use = 0;
-        g_devices[i].next = -1;
-        g_devices[i].prev = -1;
-    }
-       
-    g_first_device = -1;
-    g_last_device = -1;
-    g_num_devices = 0;
+    return 0;
 }
 
-static void init_route_table() {       
-    int i;
 
-    for (i = 0; i < MAX_ROUTES; i++) {
-        g_routes[i].use = 0;
-        g_routes[i].next = -1;
-        g_routes[i].prev = -1;
-    }
-       
-     g_first_route = -1;
-     g_last_route = -1;
-     g_num_routes = 0;
-}
 
-static void init_tables() {
-    init_link_table();
-    init_device_table();
-    init_route_table();
-    init_route_cache();
-}
+int V3_init_vnet() {
+    int i = 0;
 
-static void init_pkt_queue()
-{
-    PrintDebug("VNET Init package receiving queue\n");
+    memset(&vnet_state, 0, sizeof(vnet_state));
+       
+    INIT_LIST_HEAD(&(vnet_state.routes));
+    INIT_LIST_HEAD(&(vnet_state.devs));
 
-    g_inpkt_q = v3_create_queue();
-    v3_init_queue(g_inpkt_q);
-}
+    vnet_state.num_devs = 0;
+    vnet_state.num_routes = 0;
 
+    PrintDebug("VNET: Links and Routes tables initiated\n");
 
+    if (v3_lock_init(&(vnet_state.lock)) == -1){
+        PrintError("VNET: Failure to init lock for routes table\n");
+    }
 
-void v3_vnet_init() {  
+    PrintDebug("VNET: Locks initiated\n");
+    
+    //initial incoming pkt buffer
+    vnet_state.in_buf.start = 0;
+    vnet_state.in_buf.end = 0;
+    vnet_state.in_buf.num = 0;
 
-       PrintDebug("VNET Init: Vnet input queue successful.\n");
+    if (v3_lock_init(&(vnet_state.in_buf.lock)) == -1){
+       PrintError("VNET: add bridge, error to initiate send buf lock\n");
+    }
 
-       init_tables();
+    for (i = 0; i < BRIDGE_BUF_SIZE; i++){
+       vnet_state.in_buf.pkts[i].data = &(vnet_state.in_buf.datas[i * ETHERNET_PACKET_LEN]);
+    }
 
-       init_pkt_queue();
-       
-       //store_topologies(udp_data_socket);
+    PrintDebug("VNET: Receiving buffer initiated\n");
 
-       start_recv_data();
-}
+    vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
+
+    if (vnet_state.route_cache == NULL) {
+        PrintError("Vnet: Route Cache Init Fails\n");
+        return -1;
+    }
 
+    PrintDebug("VNET: initiated\n");
 
+    return 0;
+}