Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


Add lock to vmm_queue
[palacios.git] / palacios / src / palacios / vmm_vnet.c
index 02f5182..dffb6d4 100644 (file)
  * http://www.v3vee.org
  *
  * Copyright (c) 2009, Lei Xia <lxia@northwestern.edu> 
- * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu> 
- * Copyright (c) 2009, Jack Lange <jarusl@cs.northwestern.edu> 
- * Copyright (c) 2009, Peter Dinda <pdinda@northwestern.edu>
+ * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu>  
  * Copyright (c) 2009, The V3VEE Project <http://www.v3vee.org> 
  * All rights reserved.
  *
  * Author: Lei Xia <lxia@northwestern.edu>
  *        Yuan Tang <ytang@northwestern.edu>
- *        Jack Lange <jarusl@cs.northwestern.edu> 
- *        Peter Dinda <pdinda@northwestern.edu
  *
  * This is free software.  You are permitted to use,
  * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
  */
  
 #include <palacios/vmm_vnet.h>
+#include <palacios/vmm_hypercall.h>
+#include <palacios/vm_guest_mem.h>
+#include <palacios/vmm_lock.h>
 
-static const char any_type_str[] = "any";
-static const char not_type_str[] = "not";
-static const char none_type_str[] = "none";
-static const char empty_type_str[] = "empty";
-
-static const char link_interface_str[] = "INTERFACE";
-static const char link_edge_str[] = "EDGE";
-static const char link_any_str[] = "ANY";
-
-static const char link_prot_tcp_str[] = "tcp";
-static const char link_prot_udp_str[] = "udp";
-
-typedef enum {MAC_ANY, MAC_NOT, MAC_NONE, MAC_EMPTY} mac_type_t; //for 'src_mac_qual' and 'dst_mac_qual'
-typedef enum {LINK_INTERFACE, LINK_EDGE, LINK_ANY} link_type_t; //for 'type' and 'src_type' in struct routing
-typedef enum {TCP_TYPE, UDP_TYPE} link_prot_type_t;
+#ifndef CONFIG_DEBUG_VNET
+#undef PrintDebug
+#define PrintDebug(fmt, args...)
+#endif
 
-static const uint_t hash_key_size = 16;
-static SOCK g_udp_sockfd;
-static struct gen_queue * g_inpkt_q;
-static bool use_tcp = false;
-static uint_t vnet_udp_port = 1022;
+// 14 (ethernet frame) + 20 bytes
+struct in_pkt_header {
+    uchar_t ethernetdest[6];
+    uchar_t ethernetsrc[6];
+    uchar_t ethernettype[2]; //layer 3 protocol type
+    char ip[20];
+}__attribute__((packed));
 
-struct raw_ethernet_pkt {
-  int  size;
-  int type; // vm or link type:  INTERFACE|EDGE
-  char  data[ETHERNET_PACKET_LEN];
-};
+#define MAX_LINKS 10
+#define MAX_ROUTES 10
+#define HASH_KEY_LEN 16
+#define MIN_CACHE_SIZE 100
+#define HASH_KEY_SIZE 16
 
-//static char *vnet_version = "0.9";
-//static int vnet_server = 0;
+struct link_table {
+    struct link_entry * links[MAX_LINKS];
+    uint16_t size;
+    v3_lock_t lock;
+}__attribute__((packed));
 
+struct routing_table {
+    struct routing_entry * routes[MAX_ROUTES];
+    uint16_t size;
+    v3_lock_t lock;
+}__attribute__((packed));
 
-#define MAX_LINKS 1
-#define MAX_ROUTES 1
-#define MAX_DEVICES 16
+typedef char * route_hashkey_t;
 
-static struct topology g_links[MAX_LINKS];
-static int g_num_links; //The current number of links
-static int g_first_link;
-static int g_last_link;
+struct route_cache_entry {
+    int num_matched_routes;
+    int * matches; 
+};
 
-static struct routing g_routes[MAX_ROUTES];
-static int g_num_routes; //The current number of routes
-static int g_first_route;
-static int g_last_route;
+struct vnet_state_t {
+    struct link_table g_links;
+    struct routing_table g_routes;
+    struct gen_queue * g_inpkt_q;
+    struct hashtable * g_route_cache;
+    v3_lock_t cache_lock;
+};
 
-static struct device_list g_devices[MAX_DEVICES];
-static int g_num_devices;
-static int g_first_device;
-static int g_last_device;
+static struct vnet_state_t g_vnet_state;//global state for vnet
 
+static uint16_t ip_xsum(struct ip_header *ip_hdr, int hdr_len){
+    long sum = 0;
 
-static void print_packet(char *pkt, int size) {
-    PrintDebug("Vnet: print_data_packet: size: %d\n", size);
-    v3_hexdump(pkt, size, NULL, 0);
-}
+    uint16_t *p = (uint16_t*) ip_hdr;
+    while(hdr_len > 1){
+        sum += *(p++);
+        if(sum & 0x80000000)
+            sum = (sum & 0xFFFF) + (sum >> 16);
+            hdr_len -= 2;
+        }
+    if(hdr_len) 
+        sum += (uint16_t) *(uchar_t *)p;
+          
+    while(sum>>16)
+        sum = (sum & 0xFFFF) + (sum >> 16);
 
-#if 0
-static void print_packet_addr(char *pkt) {
-    PrintDebug("Vnet: print_packet_destination_addr: ");
-    v3_hexdump(pkt + 8, 6, NULL, 0);
-    
-    PrintDebug("Vnet: print_packet_source_addr: ");
-    v3_hexdump(pkt + 14, 6, NULL, 0);
+    return (uint16_t)~sum;
 }
 
-static void print_device_addr(char *ethaddr) {
-    PrintDebug("Vnet: print_device_addr: ");
-    v3_hexdump(ethaddr, 6, NULL, 0);
-} 
-#endif
-
-
-//network connection functions 
-static inline void raw_ethernet_packet_init(struct raw_ethernet_pkt * pt, const char * data, const size_t size) {
+static void ethernet_packet_init(struct ethernet_pkt *pt, 
+                                                              uchar_t *data, 
+                                                              const size_t size) {
     pt->size = size;
+    pt->use_header = 0;
+    memset(&pt->ext_hdr, 0, sizeof(struct udp_link_header));
     memcpy(pt->data, data, size);
 }
 
-
-
-/* Hash key format:
- * 0-5:     src_eth_addr
- * 6-11:    dest_eth_addr
- * 12:      src type
- * 13-16:   src index
- */
-typedef char * route_hashkey_t;
-
-// This is the hash value, Format: 0: num_matched_routes, 1...n: matches[] -- TY
-struct route_cache_entry {
-    int num_matched_routes;
-    int * matches; 
-};
-
-#define HASH_KEY_LEN 16
-#define MIN_CACHE_SIZE 100
-
-
-//Header of the route cache
-static struct hashtable * g_route_cache; 
-
-static uint_t hash_from_key_fn(addr_t hashkey) {    
-    uint8_t * key = (uint8_t *)hashkey;
-    return v3_hash_buffer(key, HASH_KEY_LEN);
+static inline uint_t hash_from_key_fn(addr_t hashkey) {    
+    return v3_hash_buffer((uint8_t *)hashkey, HASH_KEY_LEN);
 }
 
-static int hash_key_equal(addr_t key1, addr_t key2) {
-    uint8_t * buf1 = (uint8_t *)key1;
-    uint8_t * buf2 = (uint8_t *)key2;
-    return (memcmp(buf1, buf2, HASH_KEY_LEN) == 0);
+static inline int hash_key_equal(addr_t key1, addr_t key2) {
+    return (memcmp((uint8_t *)key1, (uint8_t *)key2, HASH_KEY_LEN) == 0);
 }
 
-static int init_route_cache() {
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, &hash_from_key_fn, &hash_key_equal);
+static int init_route_cache(struct vnet_state_t *vnet_state) { 
+    vnet_state->g_route_cache = v3_create_htable(MIN_CACHE_SIZE, 
+                                                                      &hash_from_key_fn, 
+                                                                      &hash_key_equal);
 
-    if (g_route_cache == NULL){
-        PrintError("Vnet: Route Cache Initiate Failurely\n");
+    if (vnet_state->g_route_cache == NULL) {
+        PrintError("Vnet: Route Cache Init Fails\n");
         return -1;
     }
 
     return 0;
 }
 
-static void make_hash_key(route_hashkey_t hashkey, char src_addr[6], char dest_addr[6], char src_type, int src_index) {
+static void make_hash_key(route_hashkey_t hashkey,
+                                               char src_addr[6],
+                                               char dest_addr[6],
+                                               char src_type,
+                                               int src_index) {
     int j;
 
     for (j = 0; j < 6; j++) {
@@ -161,21 +138,24 @@ static void make_hash_key(route_hashkey_t hashkey, char src_addr[6], char dest_a
 
     *(int *)(hashkey + 12) = src_index;
 }
-static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int * matches) {
+
+static int add_route_to_cache(route_hashkey_t hashkey, 
+                                                       int num_matched_r, 
+                                                       int * matches) {
     struct route_cache_entry * new_entry = NULL;
+    struct vnet_state_t *vnet_state = &g_vnet_state;
     int i;
     
     new_entry = (struct route_cache_entry *)V3_Malloc(sizeof(struct route_cache_entry));
-    if (new_entry == NULL){
+    if (new_entry == NULL) {
        PrintError("Vnet: Malloc fails\n");
        return -1;
     }
     
     new_entry->num_matched_routes = num_matched_r;
-
     new_entry->matches = (int *)V3_Malloc(num_matched_r * sizeof(int));
     
-    if (new_entry->matches == NULL){
+    if (new_entry->matches == NULL) {
        PrintError("Vnet: Malloc fails\n");
        return -1;
     }
@@ -184,9 +164,8 @@ static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int *
        new_entry->matches[i] = matches[i];
     }
     
-    //here, when v3_htable_insert return 0, it means insert fails
-    if (v3_htable_insert(g_route_cache, (addr_t)hashkey, (addr_t)new_entry) == 0){
-       PrintError("Vnet: Insert new route entry to cache failed\n");
+    if (v3_htable_insert(vnet_state->g_route_cache, (addr_t)hashkey, (addr_t)new_entry) == 0) {
+       PrintError("Vnet: Failed to insert new route entry to the cache\n");
        V3_Free(new_entry->matches);
        V3_Free(new_entry);
     }
@@ -195,11 +174,14 @@ static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int *
 }
 
 static int clear_hash_cache() {
-    v3_free_htable(g_route_cache, 1, 1);
-               
-    g_route_cache = v3_create_htable(MIN_CACHE_SIZE, hash_from_key_fn, hash_key_equal);
+    struct vnet_state_t *vnet_state = &g_vnet_state;
+
+    v3_free_htable(vnet_state->g_route_cache, 1, 1);           
+    vnet_state->g_route_cache = v3_create_htable(MIN_CACHE_SIZE, 
+                                                                        hash_from_key_fn, 
+                                                                        hash_key_equal);
     
-    if (g_route_cache == NULL){
+    if (vnet_state->g_route_cache == NULL) {
         PrintError("Vnet: Route Cache Create Failurely\n");
         return -1;
     }
@@ -208,11 +190,13 @@ static int clear_hash_cache() {
 }
 
 static int look_into_cache(route_hashkey_t hashkey, int * matches) {
+    struct route_cache_entry * found = NULL;
     int n_matches = -1;
     int i = 0;
-    struct route_cache_entry * found = NULL;
+    struct vnet_state_t *vnet_state = &g_vnet_state;
     
-    found = (struct route_cache_entry *)v3_htable_search(g_route_cache, (addr_t)hashkey);
+    found = (struct route_cache_entry *)v3_htable_search(vnet_state->g_route_cache, 
+                                                                               (addr_t)hashkey);
    
     if (found != NULL) {
         n_matches = found->num_matched_routes;
@@ -226,7 +210,12 @@ static int look_into_cache(route_hashkey_t hashkey, int * matches) {
 }
 
 
+#ifdef CONFIG_DEBUG_VNET
 
+static void print_packet(uchar_t *pkt, int size) {
+    PrintDebug("Vnet: data_packet: size: %d\n", size);
+    v3_hexdump(pkt, size, NULL, 0);
+}
 
 static inline uint8_t hex_nybble_to_nybble(const uint8_t hexnybble) {
     uint8_t x = toupper(hexnybble);
@@ -243,8 +232,7 @@ static inline uint8_t hex_byte_to_byte(const uint8_t hexbyte[2]) {
            (hex_nybble_to_nybble(hexbyte[1]) & 0xf));
 }
 
-
-static inline void string_to_mac(const char * str, uint8_t mac[6]) {
+static inline void string_to_mac(const char *str, uint8_t mac[6]) {
     int k;
 
     for (k = 0; k < 6; k++) {
@@ -252,988 +240,662 @@ static inline void string_to_mac(const char * str, uint8_t mac[6]) {
     }
 }
 
-static inline void mac_to_string(int mac[6], char * buf) {
-    snprintf(buf, 20, "%x:%x:%x:%x:%x:%x", 
+static inline void mac_to_string(char mac[6], char * buf) {
+    snprintf(buf, 20, "%02x:%02x:%02x:%02x:%02x:%02x", 
             mac[0], mac[1], mac[2],
             mac[3], mac[4], mac[5]);
 }
 
-#if 0
-static void ip_to_string(uint32_t addr, char * buf) {
-    uint32_t addr_st;
-    char * tmp_str;
-    
-    addr_st = v3_htonl(addr);
-    tmp_str = v3_inet_ntoa(addr_st);
-    
-    memcpy(buf, tmp_str, strlen(tmp_str));
-}
-#endif
-
-int find_link_by_fd(SOCK sock) {
-    int i;
 
-    FOREACH_LINK(i, g_links, g_first_link) {
-       if (g_links[i].link_sock == sock) {
-           return i;
-       }
-    }
-    
-    return -1;
-}
+static void print_route(struct routing_entry *route){
+    char dest_str[18];
+    char src_str[18];
 
-int vnet_add_link_entry(unsigned long dest, int type, int data_port,  SOCK fd) {
-    int i;
-    
-    for (i = 0; i < MAX_LINKS; i++) {
-       if (g_links[i].use == 0) {
-           g_links[i].dest = dest;
-           g_links[i].type = type;
-           g_links[i].link_sock = fd;
-           g_links[i].remote_port = data_port;
-           g_links[i].use = 1;
-
-           if (g_first_link == -1) {
-               g_first_link = i;
-           }
+    mac_to_string(route->src_mac, src_str);  
+    mac_to_string(route->dest_mac, dest_str);
 
-           g_links[i].prev = g_last_link;
-           g_links[i].next = -1;
-           
-           if (g_last_link != -1) {
-               g_links[g_last_link].next = i;
-           }
-           
-           g_last_link = i;
-           
-           g_num_links++;
-           
-           return i;
-       }
-    }
-    
-    return -1;
+    PrintDebug("SRC(%s), DEST(%s), src_mac_qual(%d), dst_mac_qual(%d)\n", 
+                 src_str, 
+                 dest_str, 
+                 route->src_mac_qual, 
+                 route->dest_mac_qual);
+    PrintDebug("Src_Link(%d), src_type(%d), dst_link(%d), dst_type(%d)\n\n", 
+                 route->src_link_idx, 
+                 route->src_type, 
+                 route->link_idx, 
+                 route->link_type);
 }
+       
 
-
-int add_sock(struct sock_list *socks, int  len, int *first_sock, int *last_sock, SOCK fd) {
+static void dump_routes(struct routing_entry **route_tables) {
     int i;
 
-    for (i = 0; i < len; i++) {
-       if (socks[i].sock == -1) {
-           socks[i].sock = fd;
-           
-           if (*first_sock == -1) {
-               *first_sock = i;
-           }
-           
-           socks[i].prev = *last_sock;
-           socks[i].next = -1;
-           
-           if (*last_sock != -1) {
-               socks[*last_sock].next = i;
-           }
-           
-           *last_sock = i;
-           
-           return i;
-       }
-    }
-    return -1;
-}
+    PrintDebug("\nVnet: route table dump start =====\n");
 
-
-int vnet_add_route_entry(char src_mac[6], char dest_mac[6], int src_mac_qual, int dest_mac_qual, int dest, int type, int src, int src_type) {
-    int i;
-    
     for(i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes[i].use == 0) {
-           
-           if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-               memcpy(g_routes[i].src_mac, src_mac, 6);
-           } else {
-               memset(g_routes[i].src_mac, 0, 6);
-           }
-           
-           if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-               memcpy(g_routes[i].dest_mac, dest_mac, 6);
-           } else {
-               memset(g_routes[i].dest_mac, 0, 6);
-           }
-           
-           g_routes[i].src_mac_qual = src_mac_qual;
-           g_routes[i].dest_mac_qual = dest_mac_qual;
-           g_routes[i].dest = dest;
-           g_routes[i].type = type;
-           g_routes[i].src = src;
-           g_routes[i].src_type = src_type;
-           g_routes[i].use = 1;
-           
-           if (g_first_route == -1) {
-               g_first_route = i;
-           }
-           
-           g_routes[i].prev = g_last_route;
-           g_routes[i].next = -1;
-           
-           if (g_last_route != -1) {
-               g_routes[g_last_route].next = i;
-           }
-           
-           g_last_route = i;
-           
-           g_num_routes++;
-           
-           return i;
-       }
+        if (route_tables[i] != NULL){
+            print_route(route_tables[i]);
+        }
     }
-    
-    clear_hash_cache();
-    
-    return -1;
-}
 
+    PrintDebug("\nVnet: route table dump end =====\n");
+}
 
-static int find_link_entry(unsigned long dest, int type) {
+static void dump_dom0_routes(struct routing_entry routes[], int size) {
     int i;
 
-    FOREACH_LINK(i, g_links, g_first_link) {
-       if ((g_links[i].dest == dest) && 
-           ((type == -1) || (g_links[i].type == type)) ) {
-           return i;
-       }
-    } 
-    
-    return -1;
-}
+    PrintDebug("\nVnet: route table from dom0 guest =====\n");
 
-static int delete_link_entry(int index) {
-    int next_i;
-    int prev_i;
-  
-    if (g_links[index].use == 0) {
-       return -1;
+    for(i = 0; i <size; i++) {
+        print_route(&routes[i]);
     }
 
-    g_links[index].dest = 0;
-    g_links[index].type = 0;
-    g_links[index].link_sock = -1;
-    g_links[index].use = 0;
+    PrintDebug("\nVnet: route table dom0 guest end =====\n");
+}
 
-    prev_i = g_links[index].prev;
-    next_i = g_links[index].next;
+static void dump_dom0_links(struct vnet_if_link links[], int size) {
+    struct vnet_if_link *link = NULL;
+    int i;
 
-    if (prev_i != -1) {
-       g_links[prev_i].next = g_links[index].next;
-    }    
+    PrintDebug("\nVnet: link table from dom0 guest =====\n");
 
-    if (next_i != -1) {
-       g_links[next_i].prev = g_links[index].prev;
+    for(i = 0; i <size; i++) {
+        link = &links[i];
+        PrintDebug("link: %d\n", i);
+        PrintDebug("dest_ip(%ld), dst_port(%d), prot_type(%d)\n", 
+                     link->dest_ip, 
+                     link->dest_port, 
+                     link->pro_type);
+        PrintDebug("vnet_header:\n");
+        v3_hexdump(&link->vnet_header, sizeof(struct udp_link_header), NULL, 0);
     }
-    
-    if (g_first_link == index) {
-       g_first_link = g_links[index].next;
-    }    
 
-    if (g_last_link == index) {
-       g_last_link = g_links[index].prev;
-    }    
-
-    g_links[index].next = -1;
-    g_links[index].prev = -1;
-    
-    g_num_links--;
-    
-    return 0;
+    PrintDebug("\nVnet: link table dom0 guest end =====\n");
 }
 
-int vnet_delete_link_entry_by_addr(unsigned long dest, int type) {
-    int index = find_link_entry(dest, type);
-  
-    if (index == -1) {
-       return -1;
-    }
-
-    return delete_link_entry(index);
-}
+#endif
 
+static int __add_link_entry(struct link_entry * link) {
+    int idx;
+    struct vnet_state_t *vnet_state = &g_vnet_state;
 
-static int find_route_entry(char src_mac[6], 
-                           char dest_mac[6], 
-                           int src_mac_qual, 
-                           int dest_mac_qual, 
-                           int dest, 
-                           int type, 
-                           int src, 
-                           int src_type) {
-    int i;
-    char temp_src_mac[6];
-    char temp_dest_mac[6];
-  
-    if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-       memcpy(temp_src_mac, src_mac, 6);
-    } else {
-       memset(temp_src_mac, 0, 6);
-    }
-    
-    if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-       memcpy(temp_dest_mac, dest_mac, 6);
-    } else {
-       memset(temp_dest_mac, 0, 6);
-    }
-    
-    FOREACH_LINK(i, g_routes, g_first_route) {
-       if ( (memcmp(temp_src_mac, g_routes[i].src_mac, 6) == 0) && 
-            (memcmp(temp_dest_mac, g_routes[i].dest_mac, 6) == 0) &&
-            (g_routes[i].src_mac_qual == src_mac_qual) &&
-            (g_routes[i].dest_mac_qual == dest_mac_qual)  &&
-            ( (type == -1) || 
-              ( (type == g_routes[i].type) && (g_routes[i].dest == dest)) ) &&
-            ( (src_type == -1) || 
-              ( (src_type == g_routes[i].src_type) && (g_routes[i].src == src)) ) ) {
-           return i;
+    v3_lock(vnet_state->g_links.lock);
+    for (idx = 0; idx < MAX_LINKS; idx++) {
+       if (vnet_state->g_links.links[idx] == NULL) {
+           vnet_state->g_links.links[idx] = link;
+           vnet_state->g_links.size++;
+           break;
        }
-    } 
-    
-    return -1;
-}
+    }
+    v3_unlock(vnet_state->g_links.lock);
 
-static int delete_route_entry(int index) {
-    int next_i;
-    int prev_i;
+    if (idx == MAX_LINKS) {
+       PrintDebug("VNET: No available Link entry for new link\n");
+       return -1;
+    }
 
-    memset(g_routes[index].src_mac, 0, 6);
-    memset(g_routes[index].dest_mac, 0, 6);
+    return idx;
+}
 
-    g_routes[index].dest = 0;
-    g_routes[index].src = 0;
-    g_routes[index].src_mac_qual = 0;
-    g_routes[index].dest_mac_qual = 0;
-    g_routes[index].type = -1;
-    g_routes[index].src_type = -1;
-    g_routes[index].use = 0;
+static int __add_route_entry(struct routing_entry * route) {
+    int idx;
+    struct vnet_state_t *vnet_state = &g_vnet_state;
 
-    prev_i = g_routes[index].prev;
-    next_i = g_routes[index].next;
-  
-    if (prev_i != -1) {
-       g_routes[prev_i].next = g_routes[index].next;
+    v3_lock(vnet_state->g_routes.lock);
+    for (idx = 0; idx < MAX_ROUTES; idx++) {
+       if (vnet_state->g_routes.routes[idx] == NULL) {
+           vnet_state->g_routes.routes[idx] = route;
+           vnet_state->g_routes.size++;
+          break;
+       }
     }
-    
-    if (next_i != -1) {
-       g_routes[next_i].prev = g_routes[index].prev;
+    v3_unlock(vnet_state->g_routes.lock);
+
+    if(idx == MAX_LINKS){
+        PrintDebug("VNET: No available route entry for new route\n");
+        return -1;
     }
-    
-    if (g_first_route == index) {
-       g_first_route = g_routes[index].next;
-    }    
 
-    if (g_last_route == index) {
-       g_last_route = g_routes[index].prev;
-    }    
+#ifdef CONFIG_DEBUG_VNET
+    dump_routes(vnet_state->g_routes.routes);
+#endif
 
-    g_routes[index].next = -1;
-    g_routes[index].prev = -1;
-    
-    g_num_routes--;
-    
-    clear_hash_cache();
-    
-    return 0;
+    return idx;
 }
 
-int vnet_delete_route_entry_by_addr(char src_mac[6], 
-                                   char dest_mac[6], 
-                                   int src_mac_qual, 
-                                   int dest_mac_qual, 
-                                   int dest, 
-                                   int type, 
-                                   int src, 
-                                   int src_type) {
-    int index = find_route_entry(src_mac, dest_mac, src_mac_qual, 
-                                dest_mac_qual, dest, type, src, src_type);
-    
-    if (index == -1) {
-       return -1;
-    }
-    
-    delete_route_entry(index);
-    
-    return 0;
-}
+int vnet_add_route_entry(char src_mac[6],
+                               char dest_mac[6],
+                               int src_mac_qual,
+                               int dest_mac_qual,
+                               int link_idx,
+                               link_type_t link_type,
+                               int src_link_idx,
+                               link_type_t src_link_type) {
+    struct routing_entry * new_route = (struct routing_entry *)V3_Malloc(sizeof(struct routing_entry));
+    int idx = -1;
 
-int delete_sock(struct sock_list * socks, int * first_sock, int * last_sock, SOCK fd) {
-    int i;
-    int prev_i;
-    int next_i;
-    
-  
-    FOREACH_SOCK(i, socks, (*first_sock)) {
-       if (socks[i].sock == fd) {
-           V3_Close_Socket(socks[i].sock);
-           socks[i].sock = -1;
-           
-           prev_i = socks[i].prev;
-           next_i = socks[i].next;
-           
-           if (prev_i != -1) {
-               socks[prev_i].next = socks[i].next;
-           }
-           
-           if (next_i != -1) {
-               socks[next_i].prev = socks[i].prev;
-           }
-           
-           if (*first_sock == i) {
-               *first_sock = socks[i].next;
-           }
-           
-           if (*last_sock == i) {
-               *last_sock = socks[i].prev;
-           }
-           
-           socks[i].next = -1;
-           socks[i].prev = -1;
+    PrintDebug("Vnet: vnet_add_route_entry\n");
+       
+    memset(new_route, 0, sizeof(struct routing_entry));
+    if ((src_mac_qual != MAC_ANY)) {
+        memcpy(new_route->src_mac, src_mac, 6);
+    }
            
-           return 0;
-       }
+    if ((dest_mac_qual != MAC_ANY)) {
+        memcpy(new_route->dest_mac, dest_mac, 6);
     }
-    return -1;
-}
-
-//setup the topology of the testing network
-static void store_topologies(SOCK fd) {
-    int i;
-    int src_mac_qual = MAC_ANY;
-    int dest_mac_qual = MAC_ANY;
-    uint_t dest;
-#ifndef VNET_SERVER
-    dest = (0 | 172 << 24 | 23 << 16 | 1 );
-    PrintDebug("VNET: store_topologies. NOT VNET_SERVER, dest = %x\n", dest);
-#else
-    dest = (0 | 172 << 24 | 23 << 16 | 2 );
-    PrintDebug("VNET: store_topologies. VNET_SERVER, dest = %x\n", dest);
-#endif
-
-    int type = UDP_TYPE;
-    int src = 0;
-    int src_type= LINK_ANY; //ANY_SRC_TYPE
-    int data_port = 22;
-    
-    //store link table
-    for (i = 0; i < MAX_LINKS; i++) {
-       if (g_links[i].use == 0) {
-           g_links[i].dest = (int)dest;
-           g_links[i].type = type;
-           g_links[i].link_sock = fd;
-           g_links[i].remote_port = data_port;
-           g_links[i].use = 1;
            
-           if (g_first_link == -1) {
-               g_first_link = i;
-           }       
+    new_route->src_mac_qual = src_mac_qual;
+    new_route->dest_mac_qual = dest_mac_qual;
+    new_route->link_idx= link_idx;
+    new_route->link_type = link_type;
+    new_route->src_link_idx = src_link_idx;
+    new_route->src_type = src_link_type;
 
-           g_links[i].prev = g_last_link;
-           g_links[i].next = -1;
-           
-           if (g_last_link != -1) {
-               g_links[g_last_link].next = i;
-           }
-           
-           g_last_link = i;
-           
-           g_num_links++;
-           PrintDebug("VNET: store_topologies. new link: socket: %d, remote %x:[%d]\n", g_links[i].link_sock, (uint_t)g_links[i].dest, g_links[i].remote_port);
-       }
+    if ((idx = __add_route_entry(new_route)) == -1) {
+       PrintError("Could not add route entry\n");
+        return -1;
     }
     
-    
-    //store route table
-    
-    type = LINK_EDGE;
-    dest = 0;
-    
-    for (i = 0; i < MAX_ROUTES; i++) {
-       if (g_routes[i].use == 0) {
-           if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
-               //                  memcpy(g_routes[i].src_mac, src_mac, 6);
-           } else {
-               memset(g_routes[i].src_mac, 0, 6);
-           }
-           
-           if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
-               //                  memcpy(g_routes[i].dest_mac, dest_mac, 6);
-           } else {
-               memset(g_routes[i].dest_mac, 0, 6);
-           }
-           
-           g_routes[i].src_mac_qual = src_mac_qual;
-           g_routes[i].dest_mac_qual = dest_mac_qual;
-           g_routes[i].dest = (int)dest;
-           g_routes[i].type = type;
-           g_routes[i].src = src;
-           g_routes[i].src_type = src_type;
-           
-           g_routes[i].use = 1;
-           
-           if (g_first_route == -1) {
-               g_first_route = i;
-           }
-           
-           g_routes[i].prev = g_last_route;
-           g_routes[i].next = -1;
-           
-           if (g_last_route != -1) {
-               g_routes[g_last_route].next = i;
-           }
-           
-           g_last_route = i;
-           
-           g_num_routes++;
-           
-           PrintDebug("VNET: store_topologies. new route: src_mac: %s, dest_mac: %s, dest: %d\n", g_routes[i].src_mac, g_routes[i].dest_mac, dest);
-           
-       }
-    }
+    clear_hash_cache();
+
+    return idx;
 }
 
-static int match_route(uint8_t * src_mac, uint8_t * dst_mac, int src_type, int src_index, int * matches) { 
-    int values[MAX_ROUTES];
+static int match_route(uint8_t * src_mac, 
+                      uint8_t * dst_mac, 
+                      link_type_t src_type, 
+                      int src_index, 
+                      int * matches) {
+    struct routing_entry *route = NULL; 
+    struct vnet_state_t *vnet_state = &g_vnet_state;
     int matched_routes[MAX_ROUTES];
-    
     int num_matches = 0;
     int i;
-    int max = 0;
-    int no = 0;
-    int exact_match = 0;
-    
-    FOREACH_ROUTE(i, g_routes, g_first_route) {
-       if ((g_routes[i].src_type != LINK_ANY) &&
-           ((g_routes[i].src_type != src_type) ||
-            ((g_routes[i].src != src_index) &&
-             (g_routes[i].src != -1)))) {
-           PrintDebug("Vnet: MatchRoute: Source route is on and does not match\n");
-           continue;
-       }
-       
-       if ( (g_routes[i].dest_mac_qual == MAC_ANY) &&
-            (g_routes[i].src_mac_qual == MAC_ANY) ) {      
-           matched_routes[num_matches] = i;
-           values[num_matches] = 3;
-           num_matches++;
-       }
-       
-       if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-           if (g_routes[i].src_mac_qual !=  MAC_NOT) {
-               if (g_routes[i].dest_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 6;
-                   
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-                   if (g_routes[i].dest_mac_qual != MAC_NOT) {   
-                       matched_routes[num_matches] = i;
-                       values[num_matches] = 8;    
-                       exact_match = 1;
-                       num_matches++;
-                   }
-               }
-           }
-       }
-       
-       if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-           if (g_routes[i].dest_mac_qual != MAC_NOT) {
-               if (g_routes[i].src_mac_qual == MAC_ANY) {
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 6;
-                   
-                   num_matches++;
-               } else if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-                   if (g_routes[i].src_mac_qual != MAC_NOT) {
-                       if (exact_match == 0) {
-                           matched_routes[num_matches] = i;
-                           values[num_matches] = 8;
-                           num_matches++;
-                       }
-                   }
-               }
-           }
-       }
-       
-       if ((g_routes[i].dest_mac_qual == MAC_NOT) &&
-           (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) != 0)) {
-           if (g_routes[i].src_mac_qual == MAC_ANY) {
-               matched_routes[num_matches] = i;
-               values[num_matches] = 5;                    
-               num_matches++;    
-           } else if (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) {
-               if (g_routes[i].src_mac_qual != MAC_NOT) {      
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 7;                  
-                   num_matches++;
-               }
-           }
-       }
-       
-       if ((g_routes[i].src_mac_qual == MAC_NOT) &&
-           (memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) != 0)) {
-           if (g_routes[i].dest_mac_qual == MAC_ANY) {
-               matched_routes[num_matches] = i;
-               values[num_matches] = 5;            
-               num_matches++;
-           } else if (memcmp((void *)&g_routes[i].dest_mac, (void *)dst_mac, 6) == 0) {
-               if (g_routes[i].dest_mac_qual != MAC_NOT) { 
-                   matched_routes[num_matches] = i;
-                   values[num_matches] = 7;
-                   num_matches++;
-               }
-           }
-       }
-    }
-    //end FOREACH_ROUTE
-    
-    FOREACH_ROUTE(i, g_routes, g_first_route) {
-       if ((memcmp((void *)&g_routes[i].src_mac, (void *)src_mac, 6) == 0) &&
-           (g_routes[i].dest_mac_qual == MAC_NONE) &&
-           ((g_routes[i].src_type == LINK_ANY) ||
-            ((g_routes[i].src_type == src_type) &&
-             ((g_routes[i].src == src_index) ||
-              (g_routes[i].src == -1))))) {
-           matched_routes[num_matches] = i;
-           values[num_matches] = 4;
-           PrintDebug("Vnet: MatchRoute: We matched a default route (%d)\n", i);
-           num_matches++;
-       }
-    }
-    
-    //If many rules have been matched, we choose one which has the highest value rating
-    if (num_matches == 0) {
-       return 0;
-    }
-    
-    for (i = 0; i < num_matches; i++) {
-       if (values[i] > max) {
-           no = 0;
-           max = values[i];
-           matches[no] = matched_routes[i];
-           no++;
-       } else if (values[i] == max) {
-           matches[no] = matched_routes[i];
-           no++;
-       }
-    }
-    
-    return no;
-}
-
-
-static int process_udpdata() {
-    struct raw_ethernet_pkt * pt;
-
-    uint32_t dest = 0;
-    uint16_t remote_port = 0;
-    SOCK link_sock = g_udp_sockfd;
-    int length = sizeof(struct raw_ethernet_pkt) - (2 * sizeof(int));   //minus the "size" and "type" 
 
-    //run in a loop to get packets from outside network, adding them to the incoming packet queue
-    while (1) {
-       pt = (struct raw_ethernet_pkt *)V3_Malloc(sizeof(struct raw_ethernet_pkt));
+#ifdef CONFIG_DEBUG_VNET
+    char dest_str[18];
+    char src_str[18];
 
-       if (pt == NULL){
-           PrintError("Vnet: process_udp: Malloc fails\n");
-           continue;
-       }
-       
-       PrintDebug("Vnet: route_thread: socket: [%d]. ready to receive from ip [%x], port [%d] or from VMs\n", link_sock, (uint_t)dest, remote_port);
-       pt->size = V3_RecvFrom_IP( link_sock, dest, remote_port, pt->data, length);
-       PrintDebug("Vnet: route_thread: socket: [%d] receive from ip [%x], port [%d]\n", link_sock, (uint_t)dest, remote_port);
-       
-       if (pt->size <= 0) {
-           PrintDebug("Vnet: process_udp: receiving packet from UDP fails\n");
-           V3_Free(pt);
-           return -1;
-       }
-       
-       PrintDebug("Vnet: process_udp: get packet\n");
-       print_packet(pt->data, pt->size);
+    mac_to_string(src_mac, src_str);  
+    mac_to_string(dst_mac, dest_str);
+    PrintDebug("Vnet: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dest_str);
+#endif
 
+    for(i = 0; i < MAX_ROUTES; i++) {
+        if (vnet_state->g_routes.routes[i] != NULL){
+            route = vnet_state->g_routes.routes[i];
+
+            if(src_type == LINK_ANY && src_index == -1) {
+                if ((route->dest_mac_qual == MAC_ANY) &&
+                      (route->src_mac_qual == MAC_ANY)) {      
+                    matched_routes[num_matches] = i;
+                    num_matches++;
+                }
        
-       //V3_Yield();
-    }
-}
-
-
-
-static int indata_handler( )
-{
-      if (!use_tcp)
-          process_udpdata();     
-
-      return 0;   
-}
+                if (memcmp((void *)&route->src_mac, (void *)src_mac, 6) == 0) {
+                    if (route->src_mac_qual !=  MAC_NOT) {
+                         if (route->dest_mac_qual == MAC_ANY) {
+                             matched_routes[num_matches] = i;
+                             num_matches++;
+                         } else if (route->dest_mac_qual != MAC_NOT &&
+                                     memcmp((void *)&route->dest_mac, (void *)dst_mac, 6) == 0) {
+                            matched_routes[num_matches] = i;
+                            num_matches++;
+                        }
+                    }
+                }
 
-static int start_recv_data()
-{
-       if (use_tcp){
-               
-       } else {
-               SOCK udp_data_socket;
-  
-               if ((udp_data_socket = V3_Create_UDP_Socket()) < 0){
-                       PrintError("VNET: Can't setup udp socket\n");
-                       return -1; 
-               }
-               PrintDebug("Vnet: vnet_setup_udp: get socket: %d\n", udp_data_socket);
-               g_udp_sockfd = udp_data_socket;
-
-               store_topologies(udp_data_socket);
-
-               if (V3_Bind_Socket(udp_data_socket, vnet_udp_port) < 0){ 
-                       PrintError("VNET: Can't bind socket\n");
-                       return -1;
-               }
-               PrintDebug("VNET: vnet_setup_udp: bind socket successful\n");
-       }
+                if (memcmp((void *)&route->dest_mac, (void *)dst_mac, 6) == 0) {
+                    if (route->dest_mac_qual != MAC_NOT) {
+                        if (route->src_mac_qual == MAC_ANY) {
+                            matched_routes[num_matches] = i;
+                            num_matches++;
+                        } else if ((route->src_mac_qual != MAC_NOT) && 
+                                       (memcmp((void *)&route->src_mac, (void *)src_mac, 6) == 0)) {
+                            matched_routes[num_matches] = i;
+                            num_matches++;
+                        }
+                     }
+                }
 
-       V3_CREATE_THREAD(&indata_handler, NULL, "VNET_DATA_HANDLER");
-       return 0;
-}
+                if ((route->dest_mac_qual == MAC_NOT) &&
+                      (memcmp((void *)&route->dest_mac, (void *)dst_mac, 6) != 0)) {
+                    if (route->src_mac_qual == MAC_ANY) {
+                        matched_routes[num_matches] = i;           
+                        num_matches++;    
+                    } else if ((route->src_mac_qual != MAC_NOT) && 
+                                   (memcmp((void *)&route->src_mac, (void *)src_mac, 6) == 0)) {     
+                        matched_routes[num_matches] = i;     
+                        num_matches++;
+                     }
+                }
 
+                if ((route->src_mac_qual == MAC_NOT) &&
+                      (memcmp((void *)&route->src_mac, (void *)src_mac, 6) != 0)) {
+                    if (route->dest_mac_qual == MAC_ANY) {
+                        matched_routes[num_matches] = i;   
+                        num_matches++;
+                    } else if ((route->dest_mac_qual != MAC_NOT) &&
+                                   (memcmp((void *)&route->dest_mac, (void *)dst_mac, 6) == 0)) {
+                        matched_routes[num_matches] = i;
+                        num_matches++;
+                    }
+                }
+            }//end if src_type == Link_any
+        }      
+    }//end for
 
-static inline int if_write_pkt(struct vnet_if_device *iface, struct raw_ethernet_pkt * pkt) {
-    return iface->input((uchar_t *)pkt->data, pkt->size);
+    PrintDebug("Vnet: match_route: Matches=%d\n", num_matches);
+       
+    for (i = 0; i < num_matches; i++) {
+        matches[i] = matched_routes[i];
+    }
+    
+    return num_matches;
 }
 
-static int handle_one_pkt(struct raw_ethernet_pkt * pkt) {
-    int src_link_index = 0;    //the value of src_link_index of udp always is 0
-    int i;
+static int handle_one_pkt(struct ethernet_pkt *pkt) {
+    int src_link_index = -1;   //the value of src_link_index of udp always is 0
     char src_mac[6];
     char dst_mac[6];
-
-    int matches[g_num_routes];
+    int matches[MAX_ROUTES];
     int num_matched_routes = 0;
+    struct in_pkt_header header;
+    char hash_key[HASH_KEY_SIZE];
+    struct vnet_state_t *vnet_state = &g_vnet_state;
+    int i;
 
-    struct HEADERS headers;
-  
     // get the ethernet and ip headers from the packet
-    memcpy((void *)&headers, (void *)pkt->data, sizeof(headers));
+    memcpy((void *)&header, (void *)pkt->data, sizeof(header));
+    memcpy(src_mac, header.ethernetsrc, 6);
+    memcpy(dst_mac, header.ethernetdest, 6);
 
-    int j;
-    for (j = 0;j < 6; j++) {
-       src_mac[j] = headers.ethernetsrc[j];
-       dst_mac[j] = headers.ethernetdest[j];
-    }
-
-
-#ifdef DEBUG
+#ifdef CONFIG_DEBUG_VNET
     char dest_str[18];
     char src_str[18];
-    
+
     mac_to_string(src_mac, src_str);  
     mac_to_string(dst_mac, dest_str);
-    
     PrintDebug("Vnet: HandleDataOverLink. SRC(%s), DEST(%s)\n", src_str, dest_str);
 #endif
-    
-    char hash_key[hash_key_size];
-    make_hash_key(hash_key, src_mac, dst_mac, LINK_EDGE, src_link_index);//link_edge -> pt->type???
-    
+
+    make_hash_key(hash_key, src_mac, dst_mac, LINK_EDGE, src_link_index); 
     num_matched_routes = look_into_cache((route_hashkey_t)hash_key, matches);
     
-    if (num_matched_routes == -1) {  //no match
-        num_matched_routes = match_route(src_mac, dst_mac, pkt->type, src_link_index, matches);
-       
-       if (num_matched_routes > 0) {
-           add_route_to_cache(hash_key, num_matched_routes,matches);      
-       }
+    if (num_matched_routes == -1) {  
+        num_matched_routes = match_route(src_mac, dst_mac, LINK_ANY, src_link_index, matches); 
+        if (num_matched_routes > 0) {
+            add_route_to_cache(hash_key, num_matched_routes,matches);      
+        }
     }
     
     PrintDebug("Vnet: HandleDataOverLink: Matches=%d\n", num_matched_routes);
+
+    if (num_matched_routes == 0) {
+        return -1;
+    }
     
-    for (i = 0; i < num_matched_routes; i++) {
+    for (i = 0; i < num_matched_routes; i++) {//send packet to all destinations
         int route_index = -1;
         int link_index = -1;
-        int dev_index = -1;
-       
+        int pkt_len = 0;
+        struct link_entry * link = NULL;
+
         route_index = matches[i];
-       
-        PrintDebug("Vnet: HandleDataOverLink: Forward packet from link according to Route entry %d\n", route_index);
-       
-        if (g_routes[route_index].type == LINK_EDGE) {
-            link_index = g_routes[route_index].dest;
-           
-            if(g_links[link_index].type == UDP_TYPE) {
-                int size;
+        link_index = vnet_state->g_routes.routes[route_index]->link_idx;
 
-                               
-                 PrintDebug("===Vnet: HandleDataOverLink: Serializing UDP Packet to link_sock [%d], dest [%x], remote_port [%d], size [%d]\n", g_links[link_index].link_sock, (uint_t)g_links[link_index].dest,  g_links[link_index].remote_port, (int)pkt->size);
+        if ((link_index < 0) || (link_index > MAX_LINKS) || 
+            (vnet_state->g_links.links[link_index] == NULL)) {
+            continue;
+        }
+       
+        link = vnet_state->g_links.links[link_index];
+        pkt_len = pkt->size;
+        if (link->type == LINK_EDGE) {
+
+             //apply the header in the beginning of the packet to be sent
+            if (link->dst_link->pro_type == UDP_TYPE) {
+                struct udp_link_header *hdr = &(link->dst_link->vnet_header);
+                struct ip_header *ip = &hdr->ip_hdr;
+                struct udp_header *udp = &hdr->udp_hdr;
+                  udp->len = pkt_len + sizeof(struct udp_header);
+                  ip->total_len = pkt_len + sizeof(struct udp_header) + sizeof(struct ip_header);
+                  ip->cksum = ip_xsum(ip, sizeof(struct ip_header));
                
-                if ((size = V3_SendTo_IP(g_links[link_index].link_sock,  g_links[link_index].dest,  g_links[link_index].remote_port, pkt->data, pkt->size)) != pkt->size)  {
-                    PrintError("Vnet: sending by UDP Exception, %x\n", size);
-                    return -1;
+                int hdr_size = sizeof(struct udp_link_header);
+                memcpy(&pkt->ext_hdr, hdr, hdr_size);
+
+                pkt_len += hdr_size;
+                if ((link->dst_link->input((uchar_t *)&pkt->ext_hdr, pkt_len, link->dst_link->private_data)) != pkt_len) {
+                    PrintDebug("VNET: Packet not sent properly to link: %d\n", link_index);
+                    continue;
                 }
-               
-                PrintDebug("Vnet: HandleDataOverLink: Serializing UDP Packet to link_sock [%d], dest [%x], remote_port [%d], size [%d]\n", g_links[link_index].link_sock, (uint_t)g_links[link_index].dest,  g_links[link_index].remote_port, (int)pkt->size);
-               
-            } else if (g_links[link_index].type == TCP_TYPE) {
-               
-            }
-        } else if (g_routes[route_index].type == LINK_INTERFACE) {
-            dev_index = g_routes[route_index].dest;
-      
-            PrintDebug("Writing Packet to device=%s\n", g_devices[dev_index].device->name);
-
-            if (if_write_pkt(g_devices[dev_index].device, pkt) == -1) {
-               PrintDebug("Can't write output packet to link\n");
-                return -1;
-            }
+            }else {
+                PrintDebug("VNET: Link protocol type not support\n");
+                continue;
+            }
+        } else if (link->type == LINK_INTERFACE) {
+            if ((link->dst_dev->input(pkt->data, pkt_len, link->dst_dev->private_data)) != pkt_len) {
+                PrintDebug("VNET: Packet not sent properly to link: %d\n", link_index);
+                continue;
+            }
         } else {
-            PrintDebug("Vnet: Wrong Edge type\n");
+            PrintDebug("Vnet: Wrong Edge type of link: %d\n", link_index);
+            continue;
         }
-    }
 
-     return 0;
+        PrintDebug("Vnet: HandleDataOverLink: Forward packet according to Route entry %d to link %d\n", route_index, link_index);
+    }
+    
+    return 0;
 }
 
-static int send_ethernet_pkt(char * buf, int length) {
-       struct raw_ethernet_pkt * pt;
+static int send_ethernet_pkt(uchar_t *data, int len, void *private_data) {
+    struct ethernet_pkt *pkt;
+    struct vnet_state_t *vnet_state = &g_vnet_state;
 
-       pt = (struct raw_ethernet_pkt *)V3_Malloc(sizeof(struct raw_ethernet_pkt));
-       raw_ethernet_packet_init(pt, buf, length);  //====here we copy sending data once 
-       
-       PrintDebug("VNET: vm_send_pkt: transmitting packet: (size:%d)\n", (int)pt->size);
-       print_packet((char *)buf, length);
-       
-       v3_enqueue(g_inpkt_q, (addr_t)pt);
-       return 0;
-       
-}
+    pkt = (struct ethernet_pkt *)V3_Malloc(sizeof(struct ethernet_pkt));
+    if(pkt == NULL){
+        PrintError("VNET: Memory allocate fails\n");
+        return -1;
+    }
 
-int v3_Send_pkt(uchar_t *buf, int length) {
-    PrintDebug("VNET: In V3_Send_pkt: pkt length %d\n", length);
-    
-    return send_ethernet_pkt((char *)buf, length);
+    memset(pkt, 0, sizeof(struct ethernet_pkt));
+    ethernet_packet_init(pkt, data, len);
+    v3_enqueue(vnet_state->g_inpkt_q, (addr_t)pkt);
+  
+#ifdef CONFIG_DEBUG_VNET
+    PrintDebug("VNET: send_pkt: transmitting packet: (size:%d)\n", (int)pkt->size);
+    print_packet((char *)data, len);
+#endif
+
+    return 0;
 }
 
-static int add_device_to_table(struct vnet_if_device*device, int type) {
-    int i;
+int v3_vnet_send_rawpkt(uchar_t * buf, 
+                                          int len, 
+                                          void *private_data) {
+    PrintDebug("VNET: In v3_vnet_send_rawpkt: pkt length %d\n", len);
     
-    for (i = 0; i < MAX_DEVICES; i++) {
-       if (g_devices[i].use == 0) {
-           g_devices[i].type = type;
-           g_devices[i].use = 1;
-           
-           if (g_first_device == -1) {
-               g_first_device = i;
-           }       
-
-           g_devices[i].prev = g_last_device;
-           g_devices[i].next = -1;
-           
-           if (g_last_device != -1) {
-               g_devices[g_last_device].next = i;
-           }
+    return send_ethernet_pkt(buf, len, private_data);
+}
 
-           g_last_device = i;
-           g_num_devices++;
-           
-           return i;
-       }
-    }
-    
-    return -1;
+//sending the packet from Dom0, should remove the link header
+int v3_vnet_send_udppkt(uchar_t * buf, 
+                                           int len, 
+                                           void *private_data) {
+    uint_t hdr_len = sizeof(struct udp_link_header);
+       
+    PrintDebug("VNET: In v3_vnet_send_udppkt: pkt length %d\n", len);
+   
+    return send_ethernet_pkt((uchar_t *)(buf+hdr_len), len - hdr_len, private_data);
 }
 
-static int search_device(char *device_name) {
+static int search_device(char * device_name) {
+    struct vnet_state_t *vnet_state = &g_vnet_state;
     int i;
 
-    for (i = 0; i < MAX_DEVICES; i++) {
-        if (g_devices[i].use == 1) {
-           if (!strcmp(device_name, g_devices[i].device->name)) {
+    for (i = 0; i < MAX_LINKS; i++) {
+        if ((vnet_state->g_links.links[i] != NULL) && (vnet_state->g_links.links[i]->type == LINK_INTERFACE)) {
+           if (strcmp(device_name, vnet_state->g_links.links[i]->dst_dev->name) == 0) {
                return i;
            }
         }
     }
-    
+
     return -1;
 }
 
-static struct vnet_if_device * delete_device_from_table(int index) {
-    int next_i;
-    int prev_i;
-    struct vnet_if_device * device = NULL;
-
-    if (g_devices[index].use == 0) {
-       return NULL;
-    }
-
-    g_devices[index].use = 0;
-
-    prev_i = g_devices[index].prev;
-    next_i = g_devices[index].next;
+int vnet_register_device(struct vm_device * vdev, 
+                        char * dev_name, 
+                        uchar_t mac[6], 
+                        int (*netif_input)(uchar_t * pkt, uint_t size, void * private_data), 
+                        void * data) {
+    struct vnet_if_device * if_dev;
 
-    if (prev_i != -1) {
-        g_devices[prev_i].next = g_devices[index].next;
-    }
-
-    if (next_i != -1) {
-        g_devices[next_i].prev = g_devices[index].prev;
-    }
-
-    if (g_first_device == index) {
-        g_first_device = g_devices[index].next;
+    int idx = search_device(dev_name);
+    if (idx != -1) {
+       PrintDebug("VNET: register device: Already has device with the name %s\n", dev_name);
+       return -1;
     }
-
-    if (g_last_device == index) {
-        g_last_device = g_devices[index].prev;
+    
+    if_dev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device)); 
+    if (if_dev == NULL) {
+       PrintError("VNET: Malloc fails\n");
+       return -1;
     }
+    
+    strcpy(if_dev->name, dev_name);
+    strncpy(if_dev->mac_addr, mac, 6);
+    if_dev->dev = vdev;
+    if_dev->input = netif_input;
+    if_dev->private_data = data;
 
-    g_devices[index].next = -1;
-    g_devices[index].prev = -1;
-
-    device = g_devices[index].device;
-    g_devices[index].device = NULL;
+    struct link_entry * link = (struct link_entry *)V3_Malloc(sizeof(struct link_entry));
+    link->type = LINK_INTERFACE;
+    link->dst_dev = if_dev;
 
-    g_num_devices--;
+    idx = __add_link_entry(link);
 
-    return device;
+    return idx;
 }
 
+int v3_vnet_pkt_process() {
+    struct ethernet_pkt * pkt;
+    struct vnet_state_t *vnet_state = &g_vnet_state;
 
-int vnet_register_device(char * dev_name, int (*netif_input)(uchar_t * pkt, uint_t size), void * data) {
-    struct vnet_if_device * dev;
-    
-    dev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device));
-    
-    if (dev == NULL){
-       PrintError("VNET: Malloc fails\n");
-       return -1;
-    }
-    
-    strncpy(dev->name, dev_name, 50);
-    dev->input = netif_input;
-    dev->data = data;
-    
-    if (add_device_to_table(dev, GENERAL_NIC) == -1) {
-       return -1;
+    while ((pkt = (struct ethernet_pkt *)v3_dequeue(vnet_state->g_inpkt_q))!= NULL) {
+        if (handle_one_pkt(pkt) != -1) {
+            PrintDebug("VNET: vnet_check: handle one packet! pt length %d, pt type %d\n", (int)pkt->size, (int)pkt->type);  
+        } else {
+            PrintDebug("VNET: vnet_check: Fail to forward one packet, discard it!\n"); 
+        }
+       
+        V3_Free(pkt); // be careful here
     }
     
     return 0;
 }
 
-int vnet_unregister_device(char * dev_name) {
+static void vnet_state_init(struct vnet_state_t *vnet_state) {
     int i;
 
-    i = search_device(dev_name);
-    
-    if (i == -1) {
-        return -1;
+    /*initial links table */
+    for (i = 0; i < MAX_LINKS; i++) {
+        vnet_state->g_links.links[i] = NULL;
     }
-
-    struct vnet_if_device * device = delete_device_from_table(i);
-
-    if (device == NULL) {
-       return -1;
+    vnet_state->g_links.size = 0;
+    if(v3_lock_init(&(vnet_state->g_links.lock)) == -1){
+        PrintError("VNET: Failure to init lock for links table\n");
     }
+    PrintDebug("VNET: Links table initiated\n");
 
-    V3_Free(device);
+    /*initial routes table */
+    for (i = 0; i < MAX_ROUTES; i++) {
+        vnet_state->g_routes.routes[i] = NULL;
+    }
+    vnet_state->g_routes.size = 0;
+    if(v3_lock_init(&(vnet_state->g_routes.lock)) == -1){
+        PrintError("VNET: Failure to init lock for routes table\n");
+    }
+    PrintDebug("VNET: Routes table initiated\n");
 
-    return 0;
+    /*initial pkt receiving queue */
+    vnet_state->g_inpkt_q = v3_create_queue();
+    v3_init_queue(vnet_state->g_inpkt_q);
+    PrintDebug("VNET: Receiving queue initiated\n");
+\r
+    /*initial routing cache */
+    init_route_cache(vnet_state);
 }
 
-int v3_Register_pkt_event(int (*netif_input)(uchar_t * pkt, uint_t size)) {
-    return vnet_register_device("NE2000", netif_input, NULL);
+static void free_link_mem(struct link_entry *link){
+    V3_Free(link->dst_dev);
+    V3_Free(link);
 }
 
-int v3_vnet_pkt_process() {
-    struct raw_ethernet_pkt * pt;
 
-    PrintDebug("VNET: In vnet_check\n");
-       
-    while ((pt = (struct raw_ethernet_pkt *)v3_dequeue(g_inpkt_q)) != NULL) {
-       PrintDebug("VNET: In vnet_check: pt length %d, pt type %d\n", (int)pt->size, (int)pt->type);
-       v3_hexdump(pt->data, pt->size, NULL, 0);
-       
-       if(handle_one_pkt(pt)) {
-           PrintDebug("VNET: vnet_check: handle one packet!\n");  
+// TODO: 
+static int addto_routing_link_tables(struct routing_entry *route_tab, 
+                                                       uint16_t num_routes,
+                                                       struct link_entry *link_tab,
+                                                       uint16_t num_links){
+    struct routing_entry *route, *new_route;
+    struct link_entry *link, *new_link;
+    int new_idx;
+    int i;
+    int link_idxs[MAX_LINKS];
+
+    //add all of the links first, record their new indexs
+    for (i = 0; i < num_links; i++) {
+       link_idxs[i] = -1;
+       link = &link_tab[i];
+       new_link = (struct link_entry *)V3_Malloc(sizeof(struct link_entry));
+
+       if (new_link == NULL){
+           PrintError("VNET: Memory allocate error\n");
+           return -1;
        }
+
+       new_link->type = link->type;
        
-       V3_Free(pt); //be careful here
-    }
-    
-    return 0;
-}
+       //TODO: how to set the input parameters here
+       if (link->type == LINK_EDGE){
+           struct vnet_if_device *ifdev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device));
 
-static void init_link_table() {
-    int i;
+           if (ifdev == NULL){
+               PrintError("VNET: Memory allocate fails\n");
+               return -1;
+           }
+           
+           memcpy(ifdev->name, link->dst_dev->name, DEVICE_NAME_LEN);
 
-    for (i = 0; i < MAX_LINKS; i++) {
-        g_links[i].use = 0;
-        g_links[i].next = -1;
-        g_links[i].prev = -1;
-    }
-       
-    g_first_link = -1;
-    g_last_link = -1;
-    g_num_links = 0;
-}
+           // TODO:
+           //ifdev->mac_addr
+           //ifdev->input
+           //ifdev->private_data
 
-static void init_device_table() {
-    int i;
+           new_link->dst_dev = ifdev;    
+       }else if (link->type == LINK_INTERFACE){
+           struct vnet_if_link *iflink = (struct vnet_if_link *)V3_Malloc(sizeof(struct vnet_if_link));
 
-    for (i = 0; i < MAX_DEVICES; i++) {
-        g_devices[i].use = 0;
-        g_devices[i].next = -1;
-        g_devices[i].prev = -1;
-    }
+           if (iflink == NULL){
+               PrintError("VNET: Memory allocate fails\n");
+               return -1;
+           }
+           iflink->pro_type = link->dst_link->pro_type;
+           iflink->dest_ip = link->dst_link->dest_ip;
+           iflink->dest_port = link->dst_link->dest_port;
+           memcpy(&iflink->vnet_header, &link->dst_link->vnet_header, sizeof(struct udp_link_header));
+
+           // TODO:
+           //iflink->input = 
+           //iflink->private_data = 
+           
+           new_link->dst_link = iflink;
+       }else{
+           PrintDebug("VNET: invalid link type\n");
+           V3_Free(new_link);
+           continue;
+       }
        
-    g_first_device = -1;
-    g_last_device = -1;
-    g_num_devices = 0;
-}
+       new_idx = __add_link_entry(new_link);
+       if (new_idx < 0) {
+           PrintError("VNET: Adding link fails\n");
+           free_link_mem(new_link);
+           continue;
+       }       
+       link_idxs[i] = new_idx;
+    }
 
-static void init_route_table() {       
-    int i;
+    //add all of routes, replace with new link indexs
+    for (i = 0; i < num_routes; i++) {
+       route = &route_tab[i];
+       if (route->link_idx < 0 || route->link_idx >= num_links || 
+           ((route->src_link_idx != -1) && 
+             (route->src_link_idx < 0 || route->src_link_idx >= num_links))){
+           PrintError("VNET: There is error in the intial tables data from guest\n");
+           continue;
+       }
 
-    for (i = 0; i < MAX_ROUTES; i++) {
-        g_routes[i].use = 0;
-        g_routes[i].next = -1;
-        g_routes[i].prev = -1;
-    }
+       new_route = (struct routing_entry *)V3_Malloc(sizeof(struct routing_entry));
+       if (new_route == NULL){
+           PrintError("VNET: Memory allocate fails\n");
+           return -1;
+       }
+       memcpy(new_route, route, sizeof(struct routing_entry));
        
-     g_first_route = -1;
-     g_last_route = -1;
-     g_num_routes = 0;
-}
+       new_route->link_idx = link_idxs[new_route->link_idx];           
+       if (route->src_link_idx != -1)
+           new_route->src_link_idx = link_idxs[new_route->src_link_idx];
+
+       if((__add_route_entry(new_route)) == -1){
+           PrintDebug("VNET: Adding route fails");
+           V3_Free(new_route);
+       }
+       new_route = NULL;       
+    }
 
-static void init_tables() {
-    init_link_table();
-    init_device_table();
-    init_route_table();
-    init_route_cache();
+    return 0;
 }
 
-static void init_pkt_queue()
-{
-    PrintDebug("VNET Init package receiving queue\n");
+//add the guest specified routes and links to the tables
+static int handle_init_tables_hcall(struct guest_info * info, uint_t hcall_id, void * priv_data) {
+    struct vnet_if_link *link_array;
+    struct routing_entry *route_array;
+    addr_t routes_addr, routes_addr_gva;
+    addr_t links_addr, links_addr_gva;
+    uint16_t route_size, link_size;
 
-    g_inpkt_q = v3_create_queue();
-    v3_init_queue(g_inpkt_q);
-}
+    routes_addr_gva = (addr_t)info->vm_regs.rbx;
+    route_size = (uint16_t)info->vm_regs.rcx;
+    links_addr_gva  = (addr_t)info->vm_regs.rdx;
+    link_size = (uint16_t)info->vm_regs.rsi;
 
+    PrintDebug("Vnet: In handle_init_tables_hcall\n");
+    PrintDebug("Vnet: route_table_start: %x, route size: %d\n", (int)routes_addr_gva, route_size);
+    PrintDebug("Vnet: link_table_start: %x, link size: %d\n", (int)links_addr_gva, link_size);
+       
+    if (guest_va_to_host_va(info, routes_addr_gva, &routes_addr) == -1) {
+       PrintError("VNET: Could not translate guest address\n");
+       return -1;
+    }
+    route_array = (struct routing_entry *)routes_addr;
 
+#ifdef CONFIG_DEBUG_VNET
+    dump_dom0_routes(route_array, route_size);
+#endif
 
-void v3_vnet_init() {  
+    if (guest_va_to_host_va(info, links_addr_gva, &links_addr) == -1) {
+       PrintError("VNET: Could not translate guest address\n");
+       return -1;
+    }  
+    link_array = (struct vnet_if_link *)links_addr;
 
-       PrintDebug("VNET Init: Vnet input queue successful.\n");
+#ifdef CONFIG_DEBUG_VNET
+    dump_dom0_links(link_array, link_size);
+#endif
+
+
+// TODO:
+    if (0)addto_routing_link_tables(route_array, route_size, NULL, 0);
 
-       init_tables();
+    return 0;
+}
 
-       init_pkt_queue();
+void v3_init_vnet() {
+    vnet_state_init(&g_vnet_state);
        
-       //store_topologies(udp_data_socket);
+    PrintDebug("VNET Initialized\n");
+}
 
-       start_recv_data();
+//only need to called in config dom0 guest
+int v3_register_guest_vnet(struct guest_info *vm){
+    return v3_register_hypercall(vm, 
+                                         VNET_INITAB_HCALL, 
+                                         handle_init_tables_hcall, 
+                                         NULL);
 }