Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


All updates on the VNET during summer
[palacios.git] / palacios / src / palacios / vmm_vnet.c
index 02f5182..1ec015e 100644 (file)
  * and the University of New Mexico.  You can find out more at 
  * http://www.v3vee.org
  *
- * Copyright (c) 2009, Lei Xia <lxia@northwestern.edu> 
- * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu> 
- * Copyright (c) 2009, Jack Lange <jarusl@cs.northwestern.edu> 
- * Copyright (c) 2009, Peter Dinda <pdinda@northwestern.edu>
+ * Copyright (c) 2010, Lei Xia <lxia@northwestern.edu> 
+ * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu>  
  * Copyright (c) 2009, The V3VEE Project <http://www.v3vee.org> 
  * All rights reserved.
  *
  * Author: Lei Xia <lxia@northwestern.edu>
  *        Yuan Tang <ytang@northwestern.edu>
- *        Jack Lange <jarusl@cs.northwestern.edu> 
- *        Peter Dinda <pdinda@northwestern.edu
  *
  * This is free software.  You are permitted to use,
  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  */
  
 #include <palacios/vmm_vnet.h>
+#include <palacios/vm_guest_mem.h>
+#include <palacios/vmm_lock.h>
+#include <palacios/vmm_queue.h>
+#include <palacios/vmm_sprintf.h>
+
+#ifndef CONFIG_DEBUG_VNET
+#undef PrintDebug
+#define PrintDebug(fmt, args...)
+#endif
 
-static const char any_type_str[] = "any";
-static const char not_type_str[] = "not";
-static const char none_type_str[] = "none";
-static const char empty_type_str[] = "empty";
-
-static const char link_interface_str[] = "INTERFACE";
-static const char link_edge_str[] = "EDGE";
-static const char link_any_str[] = "ANY";
-
-static const char link_prot_tcp_str[] = "tcp";
-static const char link_prot_udp_str[] = "udp";
-
-typedef enum {MAC_ANY, MAC_NOT, MAC_NONE, MAC_EMPTY} mac_type_t; //for 'src_mac_qual' and 'dst_mac_qual'
-typedef enum {LINK_INTERFACE, LINK_EDGE, LINK_ANY} link_type_t; //for 'type' and 'src_type' in struct routing
-typedef enum {TCP_TYPE, UDP_TYPE} link_prot_type_t;
 
-static const uint_t hash_key_size = 16;
-static SOCK g_udp_sockfd;
-static struct gen_queue * g_inpkt_q;
-static bool use_tcp = false;
-static uint_t vnet_udp_port = 1022;
+/* for UDP encapuslation */
+struct eth_header {
+    uchar_t dest[6];
+    uchar_t src[6];
+    uint16_t type;
+}__attribute__((packed));
 
-struct raw_ethernet_pkt {
-  int  size;
-  int type; // vm or link type:  INTERFACE|EDGE
-  char  data[ETHERNET_PACKET_LEN];
-};
+struct ip_header {
+    uint8_t version: 4;
+    uint8_t hdr_len: 4;
+    uchar_t tos;
+    uint16_t total_len;
+    uint16_t id;
+    uint8_t flags:     3;
+    uint16_t offset: 13;
+    uchar_t ttl;
+    uchar_t proto;
+    uint16_t cksum;
+    uint32_t src_addr;
+    uint32_t dst_addr;
+}__attribute__((packed));
 
-//static char *vnet_version = "0.9";
-//static int vnet_server = 0;
+struct udp_header {
+    uint16_t src_port;
+    uint16_t dst_port;
+    uint16_t len;
+    uint16_t csum;//set to zero, disable the xsum
+}__attribute__((packed));
 
+struct udp_link_header {
+    struct eth_header eth_hdr;
+    struct ip_header ip_hdr;
+    struct udp_header udp_hdr;
+}__attribute__((packed));
+/* end with UDP encapuslation structures */
 
-#define MAX_LINKS 1
-#define MAX_ROUTES 1
-#define MAX_DEVICES 16
 
-static struct topology g_links[MAX_LINKS];
-static int g_num_links; //The current number of links
-static int g_first_link;
-static int g_last_link;
 
-static struct routing g_routes[MAX_ROUTES];
-static int g_num_routes; //The current number of routes
-static int g_first_route;
-static int g_last_route;
 
-static struct device_list g_devices[MAX_DEVICES];
-static int g_num_devices;
-static int g_first_device;
-static int g_last_device;
+struct eth_hdr {
+    uint8_t dst_mac[6];
+    uint8_t src_mac[6];
+    uint16_t type; /* indicates layer 3 protocol type */
+} __attribute__((packed));
 
 
-static void print_packet(char *pkt, int size) {
-    PrintDebug("Vnet: print_data_packet: size: %d\n", size);
-    v3_hexdump(pkt, size, NULL, 0);
-}
+struct vnet_dev {
+    int dev_id;
+    uint8_t mac_addr[6];
+    struct v3_vm_info * vm;
+    struct v3_vnet_dev_ops dev_ops;
+    void * private_data;
 
-#if 0
-static void print_packet_addr(char *pkt) {
-    PrintDebug("Vnet: print_packet_destination_addr: ");
-    v3_hexdump(pkt + 8, 6, NULL, 0);
+    int rx_disabled;
     
-    PrintDebug("Vnet: print_packet_source_addr: ");
-    v3_hexdump(pkt + 14, 6, NULL, 0);
-}
+    struct list_head node;
+} __attribute__((packed));
 
-static void print_device_addr(char *ethaddr) {
-    PrintDebug("Vnet: print_device_addr: ");
-    v3_hexdump(ethaddr, 6, NULL, 0);
-} 
-#endif
 
+struct vnet_brg_dev {
+    struct v3_vm_info * vm;
+    struct v3_vnet_bridge_ops brg_ops;
 
-//network connection functions 
-static inline void raw_ethernet_packet_init(struct raw_ethernet_pkt * pt, const char * data, const size_t size) {
-    pt->size = size;
-    memcpy(pt->data, data, size);
-}
+    int disabled;
+    void * private_data;
+} __attribute__((packed));
 
 
 
-/* Hash key format:
- * 0-5:     src_eth_addr
- * 6-11:    dest_eth_addr
- * 12:      src type
- * 13-16:   src index
- */
-typedef char * route_hashkey_t;
+struct vnet_route_info {
+    struct v3_vnet_route route_def;
+
+    struct vnet_dev * dst_dev;
+    struct vnet_dev * src_dev;
 
-// This is the hash value, Format: 0: num_matched_routes, 1...n: matches[] -- TY
-struct route_cache_entry {
-    int num_matched_routes;
-    int * matches; 
+    struct list_head node;
+    struct list_head match_node; // used for route matching
 };
 
-#define HASH_KEY_LEN 16
-#define MIN_CACHE_SIZE 100
 
+struct route_list {
+    uint8_t hash_buf[VNET_HASH_SIZE];
 
-//Header of the route cache
-static struct hashtable * g_route_cache; 
+    uint32_t num_routes;
+    struct vnet_route_info * routes[0];
+} __attribute__((packed));
 
-static uint_t hash_from_key_fn(addr_t hashkey) {    
-    uint8_t * key = (uint8_t *)hashkey;
-    return v3_hash_buffer(key, HASH_KEY_LEN);
-}
 
-static int hash_key_equal(addr_t key1, addr_t key2) {
-    uint8_t * buf1 = (uint8_t *)key1;
-    uint8_t * buf2 = (uint8_t *)key2;
-    return (memcmp(buf1, buf2, HASH_KEY_LEN) == 0);
-}
+#define BUF_SIZE 4096
+struct pkts_buf {
+    int start, end;
+    int num; 
+    v3_lock_t lock;
+    struct v3_vnet_pkt pkts[BUF_SIZE];
+};
 
-static int init_route_cache() {
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, &hash_from_key_fn, &hash_key_equal);
 
-    if (g_route_cache == NULL){
-        PrintError("Vnet: Route Cache Initiate Failurely\n");
-        return -1;
-    }
+static struct {
+    struct list_head routes;
+    struct list_head devs;
+    
+    int num_routes;
+    int num_devs;
 
-    return 0;
-}
+    struct vnet_brg_dev *bridge;
 
-static void make_hash_key(route_hashkey_t hashkey, char src_addr[6], char dest_addr[6], char src_type, int src_index) {
-    int j;
+    v3_lock_t lock;
 
-    for (j = 0; j < 6; j++) {
-       hashkey[j] = src_addr[j];
-       hashkey[j + 6] = dest_addr[j] + 1;
-    }
+    uint8_t sidecores; /* 0 -vnet not running on sidecore, > 0, number of extra cores that can be used by VNET */
+    uint64_t cores_map; /* bitmaps for which cores can be used by VNET for sidecore, maxium 64 */
 
-    hashkey[12] = src_type;
+    struct hashtable * route_cache;
+} vnet_state;
 
-    *(int *)(hashkey + 12) = src_index;
-}
-static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int * matches) {
-    struct route_cache_entry * new_entry = NULL;
-    int i;
-    
-    new_entry = (struct route_cache_entry *)V3_Malloc(sizeof(struct route_cache_entry));
-    if (new_entry == NULL){
-       PrintError("Vnet: Malloc fails\n");
-       return -1;
-    }
-    
-    new_entry->num_matched_routes = num_matched_r;
 
-    new_entry->matches = (int *)V3_Malloc(num_matched_r * sizeof(int));
-    
-    if (new_entry->matches == NULL){
-       PrintError("Vnet: Malloc fails\n");
-       return -1;
-    }
-    
-    for (i = 0; i < num_matched_r; i++) {
-       new_entry->matches[i] = matches[i];
-    }
-    
-    //here, when v3_htable_insert return 0, it means insert fails
-    if (v3_htable_insert(g_route_cache, (addr_t)hashkey, (addr_t)new_entry) == 0){
-       PrintError("Vnet: Insert new route entry to cache failed\n");
-       V3_Free(new_entry->matches);
-       V3_Free(new_entry);
-    }
-    
-    return 0;
-}
 
-static int clear_hash_cache() {
-    v3_free_htable(g_route_cache, 1, 1);
-               
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, hash_from_key_fn, hash_key_equal);
-    
-    if (g_route_cache == NULL){
-        PrintError("Vnet: Route Cache Create Failurely\n");
-        return -1;
-    }
 
-    return 0;
+#ifdef CONFIG_DEBUG_VNET
+static inline void mac_to_string(char mac[6], char * buf) {
+    snprintf(buf, 100, "%d:%d:%d:%d:%d:%d", 
+            mac[0], mac[1], mac[2],
+            mac[3], mac[4], mac[5]);
 }
 
-static int look_into_cache(route_hashkey_t hashkey, int * matches) {
-    int n_matches = -1;
-    int i = 0;
-    struct route_cache_entry * found = NULL;
-    
-    found = (struct route_cache_entry *)v3_htable_search(g_route_cache, (addr_t)hashkey);
-   
-    if (found != NULL) {
-        n_matches = found->num_matched_routes;
-
-        for (i = 0; i < n_matches; i++) {
-            matches[i] = found->matches[i];
+static void print_route(struct vnet_route_info *route){
+    char str[50];
+
+    mac_to_string(route->route_def.src_mac, str);
+    PrintDebug("Src Mac (%s),  src_qual (%d)\n", 
+                       str, route->route_def.src_mac_qual);
+    mac_to_string(route->route_def.dst_mac, str);
+    PrintDebug("Dst Mac (%s),  dst_qual (%d)\n", 
+                       str, route->route_def.dst_mac_qual);
+    PrintDebug("Src dev id (%d), src type (%d)", 
+                       route->route_def.src_id, 
+                       route->route_def.src_type);
+    PrintDebug("Dst dev id (%d), dst type (%d)\n", 
+                       route->route_def.dst_id, 
+                       route->route_def.dst_type);
+    if (route->route_def.dst_type == LINK_INTERFACE) {
+       PrintDebug("dst_dev (%p), dst_dev_id (%d), dst_dev_ops(%p), dst_dev_data (%p)\n",
+                                       route->dst_dev,
+                                       route->dst_dev->dev_id,
+                                       (void *)&(route->dst_dev->dev_ops),
+                                       route->dst_dev->private_data);
+    }
+}
+
+static void dump_routes(){
+       struct vnet_route_info *route;
+
+       int i = 0;
+       PrintDebug("\n========Dump routes starts ============\n");
+       list_for_each_entry(route, &(vnet_state.routes), node) {
+               PrintDebug("\nroute %d:\n", ++i);
+               
+               print_route(route);
        }
-    }
-
-    return n_matches;
+       PrintDebug("\n========Dump routes end ============\n");
 }
 
+#endif
 
 
+/* 
+ * A VNET packet is a packed struct with the hashed fields grouped together.
+ * This means we can generate the hash from an offset into the pkt struct
+ */
+static inline uint_t hash_fn(addr_t hdr_ptr) {    
+    uint8_t * hdr_buf = (uint8_t *)hdr_ptr;
 
-static inline uint8_t hex_nybble_to_nybble(const uint8_t hexnybble) {
-    uint8_t x = toupper(hexnybble);
-
-    if (isdigit(x)) {
-       return x - '0';
-    } else {
-       return 10 + (x - 'A');
-    }
+    return v3_hash_buffer(hdr_buf, VNET_HASH_SIZE);
 }
 
-static inline uint8_t hex_byte_to_byte(const uint8_t hexbyte[2]) {
-    return ((hex_nybble_to_nybble(hexbyte[0]) << 4) + 
-           (hex_nybble_to_nybble(hexbyte[1]) & 0xf));
+static inline int hash_eq(addr_t key1, addr_t key2) {  
+    return (memcmp((uint8_t *)key1, (uint8_t *)key2, VNET_HASH_SIZE) == 0);
 }
 
+static int add_route_to_cache(const struct v3_vnet_pkt * pkt, struct route_list * routes) {
+    memcpy(routes->hash_buf, pkt->hash_buf, VNET_HASH_SIZE);    
 
-static inline void string_to_mac(const char * str, uint8_t mac[6]) {
-    int k;
-
-    for (k = 0; k < 6; k++) {
-       mac[k] = hex_byte_to_byte(&(str[(2 * k) + k]));
+    if (v3_htable_insert(vnet_state.route_cache, (addr_t)routes->hash_buf, (addr_t)routes) == 0) {
+       PrintError("Vnet: Failed to insert new route entry to the cache\n");
+       return -1;
     }
-}
-
-static inline void mac_to_string(int mac[6], char * buf) {
-    snprintf(buf, 20, "%x:%x:%x:%x:%x:%x", 
-            mac[0], mac[1], mac[2],
-            mac[3], mac[4], mac[5]);
-}
-
-#if 0
-static void ip_to_string(uint32_t addr, char * buf) {
-    uint32_t addr_st;
-    char * tmp_str;
-    
-    addr_st = v3_htonl(addr);
-    tmp_str = v3_inet_ntoa(addr_st);
     
-    memcpy(buf, tmp_str, strlen(tmp_str));
+    return 0;
 }
-#endif
 
-int find_link_by_fd(SOCK sock) {
-    int i;
+static int clear_hash_cache() {
 
-    FOREACH_LINK(i, g_links, g_first_link) {
-       if (g_links[i].link_sock == sock) {
-           return i;
-       }
-    }
-    
-    return -1;
+    v3_free_htable(vnet_state.route_cache, 1, 1);
+    vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
+
+    return 0;
 }
 
-int vnet_add_link_entry(unsigned long dest, int type, int data_port,  SOCK fd) {
-    int i;
+static int look_into_cache(const struct v3_vnet_pkt * pkt, struct route_list ** routes) {
     
-    for (i = 0; i < MAX_LINKS; i++) {
-       if (g_links[i].use == 0) {
-           g_links[i].dest = dest;
-           g_links[i].type = type;
-           g_links[i].link_sock = fd;
-           g_links[i].remote_port = data_port;
-           g_links[i].use = 1;
-
-           if (g_first_link == -1) {
-               g_first_link = i;
-           }
-
-           g_links[i].prev = g_last_link;
-           g_links[i].next = -1;
-           
-           if (g_last_link != -1) {
-               g_links[g_last_link].next = i;
-           }
-           
-           g_last_link = i;
-           
-           g_num_links++;
-           
-           return i;
-       }
-    }
-    
-    return -1;
+    *routes = (struct route_list *)v3_htable_search(vnet_state.route_cache, (addr_t)(pkt->hash_buf));
+   
+    return 0;
 }
 
 
-int add_sock(struct sock_list *socks, int  len, int *first_sock, int *last_sock, SOCK fd) {
-    int i;
+static struct vnet_dev * find_dev_by_id(int idx) {
+    struct vnet_dev * dev = NULL; 
 
-    for (i = 0; i < len; i++) {
-       if (socks[i].sock == -1) {
-           socks[i].sock = fd;
-           
-           if (*first_sock == -1) {
-               *first_sock = i;
-           }
-           
-           socks[i].prev = *last_sock;
-           socks[i].next = -1;
-           
-           if (*last_sock != -1) {
-               socks[*last_sock].next = i;
-           }
-           
-           *last_sock = i;
-           
-           return i;
-       }
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       int dev_id = dev->dev_id;
+
+       if (dev_id == idx)
+           return dev;
     }
-    return -1;
-}
 
+    return NULL;
+}
 
-int vnet_add_route_entry(char src_mac[6], char dest_mac[6], int src_mac_qual, int dest_mac_qual, int dest, int type, int src, int src_type) {
-    int i;
+static struct vnet_dev * find_dev_by_mac(char mac[6]) {
+    struct vnet_dev * dev = NULL; 
     
-    for(i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes[i].use == 0) {
-           
-           if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-               memcpy(g_routes[i].src_mac, src_mac, 6);
-           } else {
-               memset(g_routes[i].src_mac, 0, 6);
-           }
-           
-           if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-               memcpy(g_routes[i].dest_mac, dest_mac, 6);
-           } else {
-               memset(g_routes[i].dest_mac, 0, 6);
-           }
-           
-           g_routes[i].src_mac_qual = src_mac_qual;
-           g_routes[i].dest_mac_qual = dest_mac_qual;
-           g_routes[i].dest = dest;
-           g_routes[i].type = type;
-           g_routes[i].src = src;
-           g_routes[i].src_type = src_type;
-           g_routes[i].use = 1;
-           
-           if (g_first_route == -1) {
-               g_first_route = i;
-           }
-           
-           g_routes[i].prev = g_last_route;
-           g_routes[i].next = -1;
-           
-           if (g_last_route != -1) {
-               g_routes[g_last_route].next = i;
-           }
-           
-           g_last_route = i;
-           
-           g_num_routes++;
-           
-           return i;
-       }
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       if (!memcmp(dev->mac_addr, mac, 6))
+           return dev;
     }
-    
-    clear_hash_cache();
-    
-    return -1;
-}
 
+    return NULL;
+}
 
-static int find_link_entry(unsigned long dest, int type) {
-    int i;
+int v3_vnet_id_by_mac(char mac[6]){
 
-    FOREACH_LINK(i, g_links, g_first_link) {
-       if ((g_links[i].dest == dest) && 
-           ((type == -1) || (g_links[i].type == type)) ) {
-           return i;
-       }
-    } 
-    
-    return -1;
-}
+    struct vnet_dev *dev = find_dev_by_mac(mac);
 
-static int delete_link_entry(int index) {
-    int next_i;
-    int prev_i;
-  
-    if (g_links[index].use == 0) {
+    if (dev == NULL)
        return -1;
-    }
 
-    g_links[index].dest = 0;
-    g_links[index].type = 0;
-    g_links[index].link_sock = -1;
-    g_links[index].use = 0;
-
-    prev_i = g_links[index].prev;
-    next_i = g_links[index].next;
+    return dev->dev_id;
+}
 
-    if (prev_i != -1) {
-       g_links[prev_i].next = g_links[index].next;
-    }    
 
-    if (next_i != -1) {
-       g_links[next_i].prev = g_links[index].prev;
-    }
-    
-    if (g_first_link == index) {
-       g_first_link = g_links[index].next;
-    }    
+int v3_vnet_add_route(struct v3_vnet_route route) {
+    struct vnet_route_info * new_route = NULL;
+    unsigned long flags; 
 
-    if (g_last_link == index) {
-       g_last_link = g_links[index].prev;
-    }    
+    new_route = (struct vnet_route_info *)V3_Malloc(sizeof(struct vnet_route_info));
+    memset(new_route, 0, sizeof(struct vnet_route_info));
 
-    g_links[index].next = -1;
-    g_links[index].prev = -1;
-    
-    g_num_links--;
+    PrintDebug("Vnet: vnet_add_route_entry: dst_id: %d, dst_type: %d\n",
+                       route.dst_id, route.dst_type);  
     
-    return 0;
-}
+    memcpy(new_route->route_def.src_mac, route.src_mac, 6);
+    memcpy(new_route->route_def.dst_mac, route.dst_mac, 6);
+    new_route->route_def.src_mac_qual = route.src_mac_qual;
+    new_route->route_def.dst_mac_qual = route.dst_mac_qual;
+    new_route->route_def.dst_id = route.dst_id;
+    new_route->route_def.dst_type = route.dst_type;
+    new_route->route_def.src_id = route.src_id;
+    new_route->route_def.src_type = route.src_type;
 
-int vnet_delete_link_entry_by_addr(unsigned long dest, int type) {
-    int index = find_link_entry(dest, type);
-  
-    if (index == -1) {
-       return -1;
+    if (new_route->route_def.dst_type == LINK_INTERFACE) {
+       new_route->dst_dev = find_dev_by_id(new_route->route_def.dst_id);
     }
 
-    return delete_link_entry(index);
-}
+    if (new_route->route_def.src_type == LINK_INTERFACE) {
+       new_route->src_dev = find_dev_by_id(new_route->route_def.src_id);
+    }
 
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-static int find_route_entry(char src_mac[6], 
-                           char dest_mac[6], 
-                           int src_mac_qual, 
-                           int dest_mac_qual, 
-                           int dest, 
-                           int type, 
-                           int src, 
-                           int src_type) {
-    int i;
-    char temp_src_mac[6];
-    char temp_dest_mac[6];
-  
-    if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-       memcpy(temp_src_mac, src_mac, 6);
-    } else {
-       memset(temp_src_mac, 0, 6);
-    }
-    
-    if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-       memcpy(temp_dest_mac, dest_mac, 6);
-    } else {
-       memset(temp_dest_mac, 0, 6);
-    }
-    
-    FOREACH_LINK(i, g_routes, g_first_route) {
-       if ( (memcmp(temp_src_mac, g_routes[i].src_mac, 6) == 0) && 
-            (memcmp(temp_dest_mac, g_routes[i].dest_mac, 6) == 0) &&
-            (g_routes[i].src_mac_qual == src_mac_qual) &&
-            (g_routes[i].dest_mac_qual == dest_mac_qual)  &&
-            ( (type == -1) || 
-              ( (type == g_routes[i].type) && (g_routes[i].dest == dest)) ) &&
-            ( (src_type == -1) || 
-              ( (src_type == g_routes[i].src_type) && (g_routes[i].src == src)) ) ) {
-           return i;
-       }
-    } 
-    
-    return -1;
-}
+    list_add(&(new_route->node), &(vnet_state.routes));
+    clear_hash_cache();
 
-static int delete_route_entry(int index) {
-    int next_i;
-    int prev_i;
-
-    memset(g_routes[index].src_mac, 0, 6);
-    memset(g_routes[index].dest_mac, 0, 6);
-
-    g_routes[index].dest = 0;
-    g_routes[index].src = 0;
-    g_routes[index].src_mac_qual = 0;
-    g_routes[index].dest_mac_qual = 0;
-    g_routes[index].type = -1;
-    g_routes[index].src_type = -1;
-    g_routes[index].use = 0;
-
-    prev_i = g_routes[index].prev;
-    next_i = g_routes[index].next;
-  
-    if (prev_i != -1) {
-       g_routes[prev_i].next = g_routes[index].next;
-    }
-    
-    if (next_i != -1) {
-       g_routes[next_i].prev = g_routes[index].prev;
-    }
-    
-    if (g_first_route == index) {
-       g_first_route = g_routes[index].next;
-    }    
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+   
 
-    if (g_last_route == index) {
-       g_last_route = g_routes[index].prev;
-    }    
+#ifdef CONFIG_DEBUG_VNET
+    dump_routes();
+#endif
 
-    g_routes[index].next = -1;
-    g_routes[index].prev = -1;
-    
-    g_num_routes--;
-    
-    clear_hash_cache();
-    
     return 0;
 }
 
-int vnet_delete_route_entry_by_addr(char src_mac[6], 
-                                   char dest_mac[6], 
-                                   int src_mac_qual, 
-                                   int dest_mac_qual, 
-                                   int dest, 
-                                   int type, 
-                                   int src, 
-                                   int src_type) {
-    int index = find_route_entry(src_mac, dest_mac, src_mac_qual, 
-                                dest_mac_qual, dest, type, src, src_type);
-    
-    if (index == -1) {
-       return -1;
-    }
-    
-    delete_route_entry(index);
-    
-    return 0;
-}
 
-int delete_sock(struct sock_list * socks, int * first_sock, int * last_sock, SOCK fd) {
-    int i;
-    int prev_i;
-    int next_i;
-    
-  
-    FOREACH_SOCK(i, socks, (*first_sock)) {
-       if (socks[i].sock == fd) {
-           V3_Close_Socket(socks[i].sock);
-           socks[i].sock = -1;
-           
-           prev_i = socks[i].prev;
-           next_i = socks[i].next;
-           
-           if (prev_i != -1) {
-               socks[prev_i].next = socks[i].next;
-           }
-           
-           if (next_i != -1) {
-               socks[next_i].prev = socks[i].prev;
-           }
-           
-           if (*first_sock == i) {
-               *first_sock = socks[i].next;
-           }
-           
-           if (*last_sock == i) {
-               *last_sock = socks[i].prev;
-           }
-           
-           socks[i].next = -1;
-           socks[i].prev = -1;
-           
-           return 0;
-       }
-    }
-    return -1;
-}
 
-//setup the topology of the testing network
-static void store_topologies(SOCK fd) {
-    int i;
-    int src_mac_qual = MAC_ANY;
-    int dest_mac_qual = MAC_ANY;
-    uint_t dest;
-#ifndef VNET_SERVER
-    dest = (0 | 172 << 24 | 23 << 16 | 1 );
-    PrintDebug("VNET: store_topologies. NOT VNET_SERVER, dest = %x\n", dest);
-#else
-    dest = (0 | 172 << 24 | 23 << 16 | 2 );
-    PrintDebug("VNET: store_topologies. VNET_SERVER, dest = %x\n", dest);
-#endif
+/* At the end allocate a route_list
+ * This list will be inserted into the cache so we don't need to free it
+ */
+static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
+    struct vnet_route_info * route = NULL; 
+    struct route_list * matches = NULL;
+    int num_matches = 0;
+    int max_rank = 0;
+    struct list_head match_list;
+    struct eth_hdr * hdr = (struct eth_hdr *)(pkt->data);
+    uint8_t src_type = pkt->src_type;
+    uint32_t src_link = pkt->src_id;
 
-    int type = UDP_TYPE;
-    int src = 0;
-    int src_type= LINK_ANY; //ANY_SRC_TYPE
-    int data_port = 22;
-    
-    //store link table
-    for (i = 0; i < MAX_LINKS; i++) {
-       if (g_links[i].use == 0) {
-           g_links[i].dest = (int)dest;
-           g_links[i].type = type;
-           g_links[i].link_sock = fd;
-           g_links[i].remote_port = data_port;
-           g_links[i].use = 1;
-           
-           if (g_first_link == -1) {
-               g_first_link = i;
-           }       
+#ifdef CONFIG_DEBUG_VNET
+    {
+       char dst_str[100];
+       char src_str[100];
 
-           g_links[i].prev = g_last_link;
-           g_links[i].next = -1;
-           
-           if (g_last_link != -1) {
-               g_links[g_last_link].next = i;
-           }
-           
-           g_last_link = i;
-           
-           g_num_links++;
-           PrintDebug("VNET: store_topologies. new link: socket: %d, remote %x:[%d]\n", g_links[i].link_sock, (uint_t)g_links[i].dest, g_links[i].remote_port);
-       }
-    }
-    
-    
-    //store route table
-    
-    type = LINK_EDGE;
-    dest = 0;
-    
-    for (i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes[i].use == 0) {
-           if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-               //                  memcpy(g_routes[i].src_mac, src_mac, 6);
-           } else {
-               memset(g_routes[i].src_mac, 0, 6);
-           }
-           
-           if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-               //                  memcpy(g_routes[i].dest_mac, dest_mac, 6);
-           } else {
-               memset(g_routes[i].dest_mac, 0, 6);
-           }
-           
-           g_routes[i].src_mac_qual = src_mac_qual;
-           g_routes[i].dest_mac_qual = dest_mac_qual;
-           g_routes[i].dest = (int)dest;
-           g_routes[i].type = type;
-           g_routes[i].src = src;
-           g_routes[i].src_type = src_type;
-           
-           g_routes[i].use = 1;
-           
-           if (g_first_route == -1) {
-               g_first_route = i;
-           }
-           
-           g_routes[i].prev = g_last_route;
-           g_routes[i].next = -1;
-           
-           if (g_last_route != -1) {
-               g_routes[g_last_route].next = i;
-           }
-           
-           g_last_route = i;
-           
-           g_num_routes++;
-           
-           PrintDebug("VNET: store_topologies. new route: src_mac: %s, dest_mac: %s, dest: %d\n", g_routes[i].src_mac, g_routes[i].dest_mac, dest);
-           
-       }
+       mac_to_string(hdr->src_mac, src_str);  
+       mac_to_string(hdr->dst_mac, dst_str);
+       PrintDebug("Vnet: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
     }
-}
+#endif
 
-static int match_route(uint8_t * src_mac, uint8_t * dst_mac, int src_type, int src_index, int * matches) { 
-    int values[MAX_ROUTES];
-    int matched_routes[MAX_ROUTES];
-    
-    int num_matches = 0;
-    int i;
-    int max = 0;
-    int no = 0;
-    int exact_match = 0;
-    
-    FOREACH_ROUTE(i, g_routes, g_first_route) {
-       if ((g_routes[i].src_type != LINK_ANY) &&
-           ((g_routes[i].src_type != src_type) ||
-            ((g_routes[i].src != src_index) &&
-             (g_routes[i].src != -1)))) {
-           PrintDebug("Vnet: MatchRoute: Source route is on and does not match\n");
+    INIT_LIST_HEAD(&match_list);
+    
+#define UPDATE_MATCHES(rank) do {                              \
+       if (max_rank < (rank)) {                                \
+           max_rank = (rank);                                  \
+           INIT_LIST_HEAD(&match_list);                        \
+                                                               \
+           list_add(&(route->match_node), &match_list);        \
+           num_matches = 1;                                    \
+       } else if (max_rank == (rank)) {                        \
+           list_add(&(route->match_node), &match_list);        \
+           num_matches++;                                      \
+       }                                                       \
+    } while (0)
+    
+
+    list_for_each_entry(route, &(vnet_state.routes), node) {
+       struct v3_vnet_route * route_def = &(route->route_def);
+
+       // CHECK SOURCE TYPE HERE
+       if ( (route_def->src_type != LINK_ANY) && 
+            ( (route_def->src_type != src_type) || 
+              ( (route_def->src_id != src_link) &&
+                (route_def->src_id != (uint32_t)-1)))) {
            continue;
        }
-       
-       if ( (g_routes[i].dest_mac_qual == MAC_ANY) &&
-            (g_routes[i].src_mac_qual == MAC_ANY) ) {      
-           matched_routes[num_matches] = i;
-           values[num_matches] = 3;
-           num_matches++;
+
+
+       if ((route_def->dst_mac_qual == MAC_ANY) &&
+           (route_def->src_mac_qual == MAC_ANY)) {      
+           UPDATE_MATCHES(3);
        }
        
-       if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-           if (g_routes[i].src_mac_qual !=  MAC_NOT) {
-               if (g_routes[i].dest_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 6;
-                   
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-                   if (g_routes[i].dest_mac_qual != MAC_NOT) {   
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 8;    
-                       exact_match = 1;
-                       num_matches++;
-                   }
+       if (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) {
+           if (route_def->src_mac_qual != MAC_NOT) {
+               if (route_def->dst_mac_qual == MAC_ANY) {
+                   UPDATE_MATCHES(6);
+               } else if (route_def->dst_mac_qual != MAC_NOT &&
+                          memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
+                   UPDATE_MATCHES(8);
                }
            }
        }
-       
-       if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-           if (g_routes[i].dest_mac_qual != MAC_NOT) {
-               if (g_routes[i].src_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 6;
-                   
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-                   if (g_routes[i].src_mac_qual != MAC_NOT) {
-                       if (exact_match == 0) {
-                           matched_routes[num_matches] = i;
-                           values[num_matches] = 8;
-                           num_matches++;
-                       }
-                   }
+           
+       if (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
+           if (route_def->dst_mac_qual != MAC_NOT) {
+               if (route_def->src_mac_qual == MAC_ANY) {
+                   UPDATE_MATCHES(6);
+               } else if ((route_def->src_mac_qual != MAC_NOT) && 
+                          (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {
+                   UPDATE_MATCHES(8);
                }
            }
        }
-       
-       if ((g_routes[i].dest_mac_qual == MAC_NOT) &&
-           (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) != 0)) {
-           if (g_routes[i].src_mac_qual == MAC_ANY) {
-               matched_routes[num_matches] = i;
-               values[num_matches] = 5;                    
-               num_matches++;    
-           } else if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-               if (g_routes[i].src_mac_qual != MAC_NOT) {      
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 7;                  
-                   num_matches++;
-               }
+           
+       if ((route_def->dst_mac_qual == MAC_NOT) &&
+           (memcmp(route_def->dst_mac, hdr->dst_mac, 6) != 0)) {
+           if (route_def->src_mac_qual == MAC_ANY) {
+               UPDATE_MATCHES(5);
+           } else if ((route_def->src_mac_qual != MAC_NOT) && 
+                      (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {     
+               UPDATE_MATCHES(7);
            }
        }
        
-       if ((g_routes[i].src_mac_qual == MAC_NOT) &&
-           (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) != 0)) {
-           if (g_routes[i].dest_mac_qual == MAC_ANY) {
-               matched_routes[num_matches] = i;
-               values[num_matches] = 5;            
-               num_matches++;
-           } else if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-               if (g_routes[i].dest_mac_qual != MAC_NOT) { 
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 7;
-                   num_matches++;
-               }
+       if ((route_def->src_mac_qual == MAC_NOT) &&
+           (memcmp(route_def->src_mac, hdr->src_mac, 6) != 0)) {
+           if (route_def->dst_mac_qual == MAC_ANY) {
+               UPDATE_MATCHES(5);
+           } else if ((route_def->dst_mac_qual != MAC_NOT) &&
+                      (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0)) {
+               UPDATE_MATCHES(7);
            }
        }
+       
+       // Default route
+       if ( (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) &&
+            (route_def->dst_mac_qual == MAC_NONE)) {
+           UPDATE_MATCHES(4);
+       }
     }
-    //end FOREACH_ROUTE
-    
-    FOREACH_ROUTE(i, g_routes, g_first_route) {
-       if ((memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) &&
-           (g_routes[i].dest_mac_qual == MAC_NONE) &&
-           ((g_routes[i].src_type == LINK_ANY) ||
-            ((g_routes[i].src_type == src_type) &&
-             ((g_routes[i].src == src_index) ||
-              (g_routes[i].src == -1))))) {
-           matched_routes[num_matches] = i;
-           values[num_matches] = 4;
-           PrintDebug("Vnet: MatchRoute: We matched a default route (%d)\n", i);
-           num_matches++;
-       }
-    }
-    
-    //If many rules have been matched, we choose one which has the highest value rating
-    if (num_matches == 0) {
-       return 0;
-    }
-    
-    for (i = 0; i < num_matches; i++) {
-       if (values[i] > max) {
-           no = 0;
-           max = values[i];
-           matches[no] = matched_routes[i];
-           no++;
-       } else if (values[i] == max) {
-           matches[no] = matched_routes[i];
-           no++;
-       }
-    }
-    
-    return no;
-}
 
+    PrintDebug("Vnet: match_route: Matches=%d\n", num_matches);
 
-static int process_udpdata() {
-    struct raw_ethernet_pkt * pt;
+    if (num_matches == 0) {
+       return NULL;
+    }
 
-    uint32_t dest = 0;
-    uint16_t remote_port = 0;
-    SOCK link_sock = g_udp_sockfd;
-    int length = sizeof(struct raw_ethernet_pkt) - (2 * sizeof(int));   //minus the "size" and "type" 
+    matches = (struct route_list *)V3_Malloc(sizeof(struct route_list) + 
+                               (sizeof(struct vnet_route_info *) * num_matches));
 
-    //run in a loop to get packets from outside network, adding them to the incoming packet queue
-    while (1) {
-       pt = (struct raw_ethernet_pkt *)V3_Malloc(sizeof(struct raw_ethernet_pkt));
+    matches->num_routes = num_matches;
 
-       if (pt == NULL){
-           PrintError("Vnet: process_udp: Malloc fails\n");
-           continue;
-       }
-       
-       PrintDebug("Vnet: route_thread: socket: [%d]. ready to receive from ip [%x], port [%d] or from VMs\n", link_sock, (uint_t)dest, remote_port);
-       pt->size = V3_RecvFrom_IP( link_sock, dest, remote_port, pt->data, length);
-       PrintDebug("Vnet: route_thread: socket: [%d] receive from ip [%x], port [%d]\n", link_sock, (uint_t)dest, remote_port);
-       
-       if (pt->size <= 0) {
-           PrintDebug("Vnet: process_udp: receiving packet from UDP fails\n");
-           V3_Free(pt);
-           return -1;
+    {
+       int i = 0;
+       list_for_each_entry(route, &match_list, match_node) {
+           matches->routes[i++] = route;
        }
-       
-       PrintDebug("Vnet: process_udp: get packet\n");
-       print_packet(pt->data, pt->size);
-
-       
-       //V3_Yield();
     }
-}
 
+    return matches;
+}
 
+static int send_to_bridge(struct v3_vnet_pkt * pkt){
+    struct vnet_brg_dev *bridge = vnet_state.bridge;
 
-static int indata_handler( )
-{
-      if (!use_tcp)
-          process_udpdata();     
+    if (bridge == NULL) {
+       PrintError("VNET: No bridge to sent data to links\n");
+       return -1;
+    }
 
-      return 0;   
+    return bridge->brg_ops.input(bridge->vm, pkt, 1, bridge->private_data);
 }
 
-static int start_recv_data()
-{
-       if (use_tcp){
-               
-       } else {
-               SOCK udp_data_socket;
-  
-               if ((udp_data_socket = V3_Create_UDP_Socket()) < 0){
-                       PrintError("VNET: Can't setup udp socket\n");
-                       return -1; 
-               }
-               PrintDebug("Vnet: vnet_setup_udp: get socket: %d\n", udp_data_socket);
-               g_udp_sockfd = udp_data_socket;
-
-               store_topologies(udp_data_socket);
-
-               if (V3_Bind_Socket(udp_data_socket, vnet_udp_port) < 0){ 
-                       PrintError("VNET: Can't bind socket\n");
-                       return -1;
-               }
-               PrintDebug("VNET: vnet_setup_udp: bind socket successful\n");
-       }
 
-       V3_CREATE_THREAD(&indata_handler, NULL, "VNET_DATA_HANDLER");
+/* enable a vnet device, notify VNET it can send pkts to it */
+int v3_vnet_enable_device(int dev_id){
+    struct vnet_dev *dev = find_dev_by_id(dev_id);
+    unsigned long flags;
+
+    if(!dev)
+       return -1;
+
+    if(!dev->rx_disabled)
        return 0;
-}
 
+    flags = v3_lock_irqsave(vnet_state.lock);
+    dev->rx_disabled = 0;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+
+    /* TODO: Wake up all other guests who are trying to send pkts */
+    dev = NULL;
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       if (dev->dev_id != dev_id)
+           dev->dev_ops.start_tx(dev->private_data);
+    }
 
-static inline int if_write_pkt(struct vnet_if_device *iface, struct raw_ethernet_pkt * pkt) {
-    return iface->input((uchar_t *)pkt->data, pkt->size);
+    return 0;
 }
 
-static int handle_one_pkt(struct raw_ethernet_pkt * pkt) {
-    int src_link_index = 0;    //the value of src_link_index of udp always is 0
-    int i;
-    char src_mac[6];
-    char dst_mac[6];
+/* Notify VNET to stop sending pkts to it */
+int v3_vnet_disable_device(int dev_id){
+    struct vnet_dev *dev = find_dev_by_id(dev_id);
+    unsigned long flags;
 
-    int matches[g_num_routes];
-    int num_matched_routes = 0;
+    if(!dev)
+       return -1;
+
+    flags = v3_lock_irqsave(vnet_state.lock);
+    dev->rx_disabled = 1;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    struct HEADERS headers;
-  
-    // get the ethernet and ip headers from the packet
-    memcpy((void *)&headers, (void *)pkt->data, sizeof(headers));
 
-    int j;
-    for (j = 0;j < 6; j++) {
-       src_mac[j] = headers.ethernetsrc[j];
-       dst_mac[j] = headers.ethernetdest[j];
+    /* TODO: Notify all other guests to stop send pkts */
+    dev = NULL;
+    list_for_each_entry(dev, &(vnet_state.devs), node) {
+       if (dev->dev_id != dev_id)
+           dev->dev_ops.stop_tx(dev->private_data);
     }
 
+    return 0;
+}
 
-#ifdef DEBUG
-    char dest_str[18];
-    char src_str[18];
-    
-    mac_to_string(src_mac, src_str);  
-    mac_to_string(dst_mac, dest_str);
-    
-    PrintDebug("Vnet: HandleDataOverLink. SRC(%s), DEST(%s)\n", src_str, dest_str);
+int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
+    struct route_list * matched_routes = NULL;
+    unsigned long flags;
+    int i;
+
+#ifdef CONFIG_DEBUG_VNET
+   {
+       struct eth_hdr * hdr = (struct eth_hdr *)(pkt->header);
+       char dest_str[100];
+       char src_str[100];
+
+       mac_to_string(hdr->src_mac, src_str);  
+       mac_to_string(hdr->dst_mac, dest_str);
+       int cpu = V3_Get_CPU();
+       PrintDebug("Vnet: on cpu %d, HandleDataOverLink. SRC(%s), DEST(%s), pkt size: %d\n", cpu, src_str, dest_str, pkt->size);
+   }
 #endif
-    
-    char hash_key[hash_key_size];
-    make_hash_key(hash_key, src_mac, dst_mac, LINK_EDGE, src_link_index);//link_edge -> pt->type???
-    
-    num_matched_routes = look_into_cache((route_hashkey_t)hash_key, matches);
-    
-    if (num_matched_routes == -1) {  //no match
-        num_matched_routes = match_route(src_mac, dst_mac, pkt->type, src_link_index, matches);
-       
-       if (num_matched_routes > 0) {
-           add_route_to_cache(hash_key, num_matched_routes,matches);      
-       }
-    }
-    
-    PrintDebug("Vnet: HandleDataOverLink: Matches=%d\n", num_matched_routes);
-    
-    for (i = 0; i < num_matched_routes; i++) {
-        int route_index = -1;
-        int link_index = -1;
-        int dev_index = -1;
+
+    flags = v3_lock_irqsave(vnet_state.lock);
+
+    look_into_cache(pkt, &matched_routes);
        
-        route_index = matches[i];
+    if (matched_routes == NULL) {  
+       PrintError("Vnet: send pkt Looking into routing table\n");
        
-        PrintDebug("Vnet: HandleDataOverLink: Forward packet from link according to Route entry %d\n", route_index);
+       matched_routes = match_route(pkt);
        
-        if (g_routes[route_index].type == LINK_EDGE) {
-            link_index = g_routes[route_index].dest;
-           
-            if(g_links[link_index].type == UDP_TYPE) {
-                int size;
-
-                               
-                 PrintDebug("===Vnet: HandleDataOverLink: Serializing UDP Packet to link_sock [%d], dest [%x], remote_port [%d], size [%d]\n", g_links[link_index].link_sock, (uint_t)g_links[link_index].dest,  g_links[link_index].remote_port, (int)pkt->size);
-               
-                if ((size = V3_SendTo_IP(g_links[link_index].link_sock,  g_links[link_index].dest,  g_links[link_index].remote_port, pkt->data, pkt->size)) != pkt->size)  {
-                    PrintError("Vnet: sending by UDP Exception, %x\n", size);
-                    return -1;
-                }
-               
-                PrintDebug("Vnet: HandleDataOverLink: Serializing UDP Packet to link_sock [%d], dest [%x], remote_port [%d], size [%d]\n", g_links[link_index].link_sock, (uint_t)g_links[link_index].dest,  g_links[link_index].remote_port, (int)pkt->size);
-               
-            } else if (g_links[link_index].type == TCP_TYPE) {
-               
-            }
-        } else if (g_routes[route_index].type == LINK_INTERFACE) {
-            dev_index = g_routes[route_index].dest;
-      
-            PrintDebug("Writing Packet to device=%s\n", g_devices[dev_index].device->name);
-
-            if (if_write_pkt(g_devices[dev_index].device, pkt) == -1) {
-               PrintDebug("Can't write output packet to link\n");
-                return -1;
-            }
-        } else {
-            PrintDebug("Vnet: Wrong Edge type\n");
-        }
+       if (matched_routes) {
+           add_route_to_cache(pkt, matched_routes);
+       } else {
+           PrintDebug("Could not find route for packet... discards packet\n");
+           v3_unlock_irqrestore(vnet_state.lock, flags);
+           return 0; /* do we return -1 here?*/
+       }
     }
 
-     return 0;
-}
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-static int send_ethernet_pkt(char * buf, int length) {
-       struct raw_ethernet_pkt * pt;
+    PrintDebug("Vnet: send pkt route matches %d\n", matched_routes->num_routes);
 
-       pt = (struct raw_ethernet_pkt *)V3_Malloc(sizeof(struct raw_ethernet_pkt));
-       raw_ethernet_packet_init(pt, buf, length);  //====here we copy sending data once 
+    for (i = 0; i < matched_routes->num_routes; i++) {
+        struct vnet_route_info * route = matched_routes->routes[i];
        
-       PrintDebug("VNET: vm_send_pkt: transmitting packet: (size:%d)\n", (int)pt->size);
-       print_packet((char *)buf, length);
-       
-       v3_enqueue(g_inpkt_q, (addr_t)pt);
-       return 0;
-       
-}
+        if (route->route_def.dst_type == LINK_EDGE) {                  
+            pkt->dst_type = LINK_EDGE;
+            pkt->dst_id = route->route_def.dst_id;
+
+            if (send_to_bridge(pkt) == -1) {
+                PrintDebug("VNET: Packet not sent properly to bridge\n");
+                continue;
+            }         
+        } else if (route->route_def.dst_type == LINK_INTERFACE) {
+            if (!route->dst_dev->rx_disabled){ 
+                 if(route->dst_dev->dev_ops.input(route->dst_dev->vm, pkt, route->dst_dev->private_data) == -1) {
+                       PrintDebug("VNET: Packet not sent properly\n");
+                       continue;
+                 }
+            }
+        } else {
+            PrintError("VNET: Wrong Edge type\n");
+        }
 
-int v3_Send_pkt(uchar_t *buf, int length) {
-    PrintDebug("VNET: In V3_Send_pkt: pkt length %d\n", length);
+        PrintDebug("VNET: Forward one packet according to Route %d\n", i);
+    }
     
-    return send_ethernet_pkt((char *)buf, length);
+    return 0;
 }
 
-static int add_device_to_table(struct vnet_if_device*device, int type) {
-    int i;
-    
-    for (i = 0; i < MAX_DEVICES; i++) {
-       if (g_devices[i].use == 0) {
-           g_devices[i].type = type;
-           g_devices[i].use = 1;
-           
-           if (g_first_device == -1) {
-               g_first_device = i;
-           }       
+int v3_vnet_add_dev(struct v3_vm_info *vm, uint8_t mac[6], 
+                   struct v3_vnet_dev_ops *ops,
+                   void * priv_data){
+    struct vnet_dev * new_dev = NULL;
+    unsigned long flags;
 
-           g_devices[i].prev = g_last_device;
-           g_devices[i].next = -1;
-           
-           if (g_last_device != -1) {
-               g_devices[g_last_device].next = i;
-           }
+    new_dev = (struct vnet_dev *)V3_Malloc(sizeof(struct vnet_dev)); 
 
-           g_last_device = i;
-           g_num_devices++;
-           
-           return i;
-       }
+    if (new_dev == NULL) {
+       PrintError("VNET: Malloc fails\n");
+       return -1;
     }
-    
-    return -1;
-}
+   
+    memcpy(new_dev->mac_addr, mac, 6);
+    new_dev->dev_ops.input = ops->input;
+    new_dev->dev_ops.poll = ops->poll;
+    new_dev->private_data = priv_data;
+    new_dev->vm = vm;
+    new_dev->dev_id = 0;       
 
-static int search_device(char *device_name) {
-    int i;
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    for (i = 0; i < MAX_DEVICES; i++) {
-        if (g_devices[i].use == 1) {
-           if (!strcmp(device_name, g_devices[i].device->name)) {
-               return i;
-           }
-        }
+    if (!find_dev_by_mac(mac)) {
+       list_add(&(new_dev->node), &(vnet_state.devs));
+       new_dev->dev_id = ++vnet_state.num_devs;
     }
-    
-    return -1;
-}
 
-static struct vnet_if_device * delete_device_from_table(int index) {
-    int next_i;
-    int prev_i;
-    struct vnet_if_device * device = NULL;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    if (g_devices[index].use == 0) {
-       return NULL;
+    /* if the device was found previosly the id should still be 0 */
+    if (new_dev->dev_id == 0) {
+       PrintError("Device Alrady exists\n");
+       return -1;
     }
 
-    g_devices[index].use = 0;
+    PrintDebug("Vnet: Add Device: dev_id %d\n", new_dev->dev_id);
 
-    prev_i = g_devices[index].prev;
-    next_i = g_devices[index].next;
+    return new_dev->dev_id;
+}
 
-    if (prev_i != -1) {
-        g_devices[prev_i].next = g_devices[index].next;
-    }
 
-    if (next_i != -1) {
-        g_devices[next_i].prev = g_devices[index].prev;
-    }
+/* TODO: Still need to figure out how to handle this multicore part --Lei
+  */
+void  v3_vnet_poll(struct v3_vm_info *vm){
+    struct vnet_dev * dev = NULL; 
 
-    if (g_first_device == index) {
-        g_first_device = g_devices[index].next;
-    }
-
-    if (g_last_device == index) {
-        g_last_device = g_devices[index].prev;
+    switch (vnet_state.sidecores) {
+       case 0:
+               list_for_each_entry(dev, &(vnet_state.devs), node) {
+                   if(dev->vm == vm){
+                       dev->dev_ops.poll(vm, dev->private_data);
+                   }
+              }
+               break;
+       case 1:
+               break;
+       case 2:
+           list_for_each_entry(dev, &(vnet_state.devs), node) {
+               int cpu_id = vm->cores[0].cpu_id + 2; /* temporary here, should use vnet_state.cores_map */
+                struct v3_vnet_dev_xcall_args dev_args; /* could cause problem here -LX */
+                dev_args.vm = vm;
+               dev_args.private_data = dev->private_data;
+               V3_Call_On_CPU(cpu_id, dev->dev_ops.poll_xcall, (void *)&dev_args);
+           }
+           break;
+       default:
+           break;
     }
+}
 
-    g_devices[index].next = -1;
-    g_devices[index].prev = -1;
-
-    device = g_devices[index].device;
-    g_devices[index].device = NULL;
-
-    g_num_devices--;
+int v3_vnet_add_bridge(struct v3_vm_info * vm,
+                      struct v3_vnet_bridge_ops *ops,
+                      void * priv_data) {
+    unsigned long flags;
+    int bridge_free = 0;
+    struct vnet_brg_dev * tmp_bridge = NULL;    
+    
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    return device;
-}
+    if (vnet_state.bridge == NULL) {
+       bridge_free = 1;
+       vnet_state.bridge = (void *)1;
+    }
 
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-int vnet_register_device(char * dev_name, int (*netif_input)(uchar_t * pkt, uint_t size), void * data) {
-    struct vnet_if_device * dev;
-    
-    dev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device));
-    
-    if (dev == NULL){
-       PrintError("VNET: Malloc fails\n");
+    if (bridge_free == 0) {
+       PrintError("Bridge already set\n");
        return -1;
     }
-    
-    strncpy(dev->name, dev_name, 50);
-    dev->input = netif_input;
-    dev->data = data;
-    
-    if (add_device_to_table(dev, GENERAL_NIC) == -1) {
+
+    tmp_bridge = (struct vnet_brg_dev *)V3_Malloc(sizeof(struct vnet_brg_dev));
+
+    if (tmp_bridge == NULL) {
+       PrintError("Malloc Fails\n");
+       vnet_state.bridge = NULL;
        return -1;
     }
     
+    tmp_bridge->vm = vm;
+    tmp_bridge->brg_ops.input = ops->input;
+    tmp_bridge->brg_ops.xcall_input = ops->xcall_input;
+    tmp_bridge->brg_ops.polling_pkt = ops->polling_pkt;
+    tmp_bridge->private_data = priv_data;
+    tmp_bridge->disabled = 0;
+       
+    /* make this atomic to avoid possible race conditions */
+    flags = v3_lock_irqsave(vnet_state.lock);
+    vnet_state.bridge = tmp_bridge;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
+
     return 0;
 }
 
-int vnet_unregister_device(char * dev_name) {
-    int i;
 
-    i = search_device(dev_name);
+#if 0
+int v3_vnet_disable_bridge() {
+    unsigned long flags; 
     
-    if (i == -1) {
-        return -1;
-    }
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-    struct vnet_if_device * device = delete_device_from_table(i);
-
-    if (device == NULL) {
-       return -1;
+    if (vnet_state.bridge != NULL) {
+       vnet_state.bridge->disabled = 1;
     }
 
-    V3_Free(device);
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
     return 0;
 }
 
-int v3_Register_pkt_event(int (*netif_input)(uchar_t * pkt, uint_t size)) {
-    return vnet_register_device("NE2000", netif_input, NULL);
-}
-
-int v3_vnet_pkt_process() {
-    struct raw_ethernet_pkt * pt;
 
-    PrintDebug("VNET: In vnet_check\n");
-       
-    while ((pt = (struct raw_ethernet_pkt *)v3_dequeue(g_inpkt_q)) != NULL) {
-       PrintDebug("VNET: In vnet_check: pt length %d, pt type %d\n", (int)pt->size, (int)pt->type);
-       v3_hexdump(pt->data, pt->size, NULL, 0);
-       
-       if(handle_one_pkt(pt)) {
-           PrintDebug("VNET: vnet_check: handle one packet!\n");  
-       }
-       
-       V3_Free(pt); //be careful here
-    }
+int v3_vnet_enable_bridge() {
+    unsigned long flags; 
     
-    return 0;
-}
+    flags = v3_lock_irqsave(vnet_state.lock);
 
-static void init_link_table() {
-    int i;
-
-    for (i = 0; i < MAX_LINKS; i++) {
-        g_links[i].use = 0;
-        g_links[i].next = -1;
-        g_links[i].prev = -1;
+    if (vnet_state.bridge != NULL) {
+       vnet_state.bridge->disabled = 0;
     }
-       
-    g_first_link = -1;
-    g_last_link = -1;
-    g_num_links = 0;
-}
 
-static void init_device_table() {
-    int i;
+    v3_unlock_irqrestore(vnet_state.lock, flags);
 
-    for (i = 0; i < MAX_DEVICES; i++) {
-        g_devices[i].use = 0;
-        g_devices[i].next = -1;
-        g_devices[i].prev = -1;
-    }
-       
-    g_first_device = -1;
-    g_last_device = -1;
-    g_num_devices = 0;
+    return 0;
 }
+#endif
 
-static void init_route_table() {       
-    int i;
-
-    for (i = 0; i < MAX_ROUTES; i++) {
-        g_routes[i].use = 0;
-        g_routes[i].next = -1;
-        g_routes[i].prev = -1;
-    }
+int v3_init_vnet() {
+    memset(&vnet_state, 0, sizeof(vnet_state));
        
-     g_first_route = -1;
-     g_last_route = -1;
-     g_num_routes = 0;
-}
+    INIT_LIST_HEAD(&(vnet_state.routes));
+    INIT_LIST_HEAD(&(vnet_state.devs));
 
-static void init_tables() {
-    init_link_table();
-    init_device_table();
-    init_route_table();
-    init_route_cache();
-}
-
-static void init_pkt_queue()
-{
-    PrintDebug("VNET Init package receiving queue\n");
-
-    g_inpkt_q = v3_create_queue();
-    v3_init_queue(g_inpkt_q);
-}
+    vnet_state.num_devs = 0;
+    vnet_state.num_routes = 0;
 
+    PrintDebug("VNET: Links and Routes tables initiated\n");
 
+    if (v3_lock_init(&(vnet_state.lock)) == -1){
+        PrintError("VNET: Failure to init lock for routes table\n");
+    }
+    PrintDebug("VNET: Locks initiated\n");
 
-void v3_vnet_init() {  
+    vnet_state.route_cache = v3_create_htable(0, &hash_fn, &hash_eq);
 
-       PrintDebug("VNET Init: Vnet input queue successful.\n");
+    if (vnet_state.route_cache == NULL) {
+        PrintError("Vnet: Route Cache Init Fails\n");
+        return -1;
+    }
 
-       init_tables();
+    vnet_state.sidecores = 0;
+    vnet_state.cores_map = 0;
 
-       init_pkt_queue();
-       
-       //store_topologies(udp_data_socket);
+    PrintDebug("VNET: initiated\n");
 
-       start_recv_data();
+    return 0;
 }
-
-