2 * This file is part of the Palacios Virtual Machine Monitor developed
3 * by the V3VEE Project with funding from the United States National
4 * Science Foundation and the Department of Energy.
6 * The V3VEE Project is a joint project between Northwestern University
7 * and the University of New Mexico. You can find out more at
10 * Copyright (c) 2009, Lei Xia <lxia@northwestern.edu>
11 * Copyright (c) 2009, Yuan Tang <ytang@northwestern.edu>
12 * Copyright (c) 2009, Jack Lange <jarusl@cs.northwestern.edu>
13 * Copyright (c) 2009, Peter Dinda <pdinda@northwestern.edu>
14 * Copyright (c) 2009, The V3VEE Project <http://www.v3vee.org>
15 * All rights reserved.
17 * Author: Lei Xia <lxia@northwestern.edu>
18 * Yuan Tang <ytang@northwestern.edu>
19 * Jack Lange <jarusl@cs.northwestern.edu>
20 * Peter Dinda <pdinda@northwestern.edu
22 * This is free software. You are permitted to use,
23 * redistribute, and modify it as specified in the file "V3VEE_LICENSE".
26 #include <palacios/vmm_vnet.h>
27 #include <palacios/vmm_hypercall.h>
28 #include <palacios/vm_guest_mem.h>
30 #ifndef CONFIG_DEBUG_VNET
32 #define PrintDebug(fmt, args...)
39 char data[ETHERNET_PACKET_LEN];
43 // 14 (ethernet frame) + 20 bytes
44 struct in_pkt_header {
47 unsigned char ethernettype[2]; // indicates layer 3 protocol type
51 #define VNET_INITAB_HCALL 0xca00 // inital hypercall id
55 #define HASH_KEY_LEN 16
56 #define MIN_CACHE_SIZE 100
57 static const uint_t hash_key_size = 16;
60 struct link_entry *links[MAX_LINKS];
64 struct routing_table {
65 struct routing_entry *routes[MAX_ROUTES];
69 static struct link_table g_links;
70 static struct routing_table g_routes;
71 static struct gen_queue *g_inpkt_q;
79 typedef char *route_hashkey_t;
81 // This is the hash value, Format: 0: num_matched_routes, 1...n: matches[] -- TY
82 struct route_cache_entry {
83 int num_matched_routes;
88 static struct hashtable *g_route_cache;
91 static void print_packet(char *pkt, int size) {
92 PrintDebug("Vnet: print_data_packet: size: %d\n", size);
93 v3_hexdump(pkt, size, NULL, 0);
97 static void print_packet_addr(char *pkt) {
98 PrintDebug("Vnet: print_packet_destination_addr: ");
99 v3_hexdump(pkt + 8, 6, NULL, 0);
101 PrintDebug("Vnet: print_packet_source_addr: ");
102 v3_hexdump(pkt + 14, 6, NULL, 0);
105 static void print_device_addr(char *ethaddr) {
106 PrintDebug("Vnet: print_device_addr: ");
107 v3_hexdump(ethaddr, 6, NULL, 0);
112 //network connection functions
113 static inline void ethernet_packet_init(struct ethernet_pkt *pt, const char *data, const size_t size)
116 memcpy(pt->data, data, size);
119 static uint_t hash_from_key_fn(addr_t hashkey)
121 uint8_t * key = (uint8_t *)hashkey;
122 return v3_hash_buffer(key, HASH_KEY_LEN);
125 static int hash_key_equal(addr_t key1, addr_t key2)
127 uint8_t * buf1 = (uint8_t *)key1;
128 uint8_t * buf2 = (uint8_t *)key2;
129 return (memcmp(buf1, buf2, HASH_KEY_LEN) == 0);
132 static int init_route_cache()
134 g_route_cache = v3_create_htable(MIN_CACHE_SIZE, &hash_from_key_fn, &hash_key_equal);
136 if (g_route_cache == NULL){
137 PrintError("Vnet: Route Cache Initiate Failurely\n");
144 static void make_hash_key(route_hashkey_t hashkey,
152 for (j = 0; j < 6; j++) {
153 hashkey[j] = src_addr[j];
154 hashkey[j + 6] = dest_addr[j] + 1;
157 hashkey[12] = src_type;
159 *(int *)(hashkey + 12) = src_index;
162 static int add_route_to_cache(route_hashkey_t hashkey, int num_matched_r, int *matches)
164 struct route_cache_entry * new_entry = NULL;
167 new_entry = (struct route_cache_entry *)V3_Malloc(sizeof(struct route_cache_entry));
168 if (new_entry == NULL){
169 PrintError("Vnet: Malloc fails\n");
173 new_entry->num_matched_routes = num_matched_r;
175 new_entry->matches = (int *)V3_Malloc(num_matched_r * sizeof(int));
177 if (new_entry->matches == NULL){
178 PrintError("Vnet: Malloc fails\n");
182 for (i = 0; i < num_matched_r; i++) {
183 new_entry->matches[i] = matches[i];
186 //here, when v3_htable_insert return 0, it means insert fails
187 if (v3_htable_insert(g_route_cache, (addr_t)hashkey, (addr_t)new_entry) == 0){
188 PrintError("Vnet: Insert new route entry to cache failed\n");
189 V3_Free(new_entry->matches);
196 static int clear_hash_cache()
198 v3_free_htable(g_route_cache, 1, 1);
200 g_route_cache = v3_create_htable(MIN_CACHE_SIZE, hash_from_key_fn, hash_key_equal);
202 if (g_route_cache == NULL){
203 PrintError("Vnet: Route Cache Create Failurely\n");
210 static int look_into_cache(route_hashkey_t hashkey, int *matches)
214 struct route_cache_entry * found = NULL;
216 found = (struct route_cache_entry *)v3_htable_search(g_route_cache, (addr_t)hashkey);
219 n_matches = found->num_matched_routes;
221 for (i = 0; i < n_matches; i++) {
222 matches[i] = found->matches[i];
229 static inline uint8_t hex_nybble_to_nybble(const uint8_t hexnybble)
231 uint8_t x = toupper(hexnybble);
236 return 10 + (x - 'A');
240 static inline uint8_t hex_byte_to_byte(const uint8_t hexbyte[2])
242 return ((hex_nybble_to_nybble(hexbyte[0]) << 4) +
243 (hex_nybble_to_nybble(hexbyte[1]) & 0xf));
246 static inline void string_to_mac(const char *str, uint8_t mac[6])
250 for (k = 0; k < 6; k++) {
251 mac[k] = hex_byte_to_byte(&(str[(2 * k) + k]));
255 static inline void mac_to_string(char mac[6], char * buf)
257 snprintf(buf, 20, "%x:%x:%x:%x:%x:%x",
258 mac[0], mac[1], mac[2],
259 mac[3], mac[4], mac[5]);
262 static int add_link_entry(struct link_entry *link)
266 for (idx = 0; idx < MAX_LINKS; idx++) {
267 if (g_links.links[idx] == NULL) {
268 g_links.links[idx] = link;
278 static int add_route_entry(struct routing_entry *route)
282 for (idx = 0; idx < MAX_ROUTES; idx++) {
283 if (g_routes.routes[idx] == NULL) {
284 g_routes.routes[idx] = route;
295 int vnet_add_route_entry(char src_mac[6],
300 link_type_t link_type,
302 link_type_t src_type)
304 struct routing_entry *new_route = (struct routing_entry *)V3_Malloc(sizeof(struct routing_entry));
306 memset(new_route, 0, sizeof(struct routing_entry));
308 if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
309 memcpy(new_route->src_mac, src_mac, 6);
312 if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
313 memcpy(new_route->dest_mac, dest_mac, 6);
316 new_route->src_mac_qual = src_mac_qual;
317 new_route->dest_mac_qual = dest_mac_qual;
318 new_route->link_idx= link_idx;
319 new_route->link_type = link_type;
320 new_route->src_link_idx = src;
321 new_route->src_type = src_type;
324 if ((idx = add_route_entry(new_route)) == -1)
332 static void * delete_link_entry(int index)
334 struct link_entry *link = NULL;
338 if (index >= MAX_LINKS || g_links.links[index] == NULL) {
342 link = g_links.links[index];
343 type = g_links.links[index]->type;
345 if (type == LINK_INTERFACE)
346 ret = (void *)g_links.links[index]->dst_dev;
347 else if (type == LINK_EDGE)
348 ret = (void *)g_links.links[index]->dst_link;
350 g_links.links[index] = NULL;
358 static int find_route_entry(char src_mac[6],
363 link_type_t link_type,
365 link_type_t src_type)
368 char temp_src_mac[6];
369 char temp_dest_mac[6];
371 if ((src_mac_qual != MAC_ANY) && (src_mac_qual != MAC_NONE)) {
372 memcpy(temp_src_mac, src_mac, 6);
374 memset(temp_src_mac, 0, 6);
377 if ((dest_mac_qual != MAC_ANY) && (dest_mac_qual != MAC_NONE)) {
378 memcpy(temp_dest_mac, dest_mac, 6);
380 memset(temp_dest_mac, 0, 6);
383 for (i = 0; i<MAX_ROUTES; i++) {
384 if (g_routes.routes[i] != NULL) {
385 if ((memcmp(temp_src_mac, g_routes.routes[i]->src_mac, 6) == 0) &&
386 (memcmp(temp_dest_mac, g_routes.routes[i]->dest_mac, 6) == 0) &&
387 (g_routes.routes[i]->src_mac_qual == src_mac_qual) &&
388 (g_routes.routes[i]->dest_mac_qual == dest_mac_qual) &&
389 ((link_type == LINK_ANY) ||
390 ((link_type == g_routes.routes[i]->link_type) && (g_routes.routes[i]->link_idx == link_idx))) &&
391 ((src_type == LINK_ANY) ||
392 ((src_type == g_routes.routes[i]->src_type) && (g_routes.routes[i]->src_link_idx == src)))) {
401 static int delete_route_entry(int index)
403 struct routing_entry *route;
405 if (index >= MAX_ROUTES || g_routes.routes[index] == NULL)
408 route = g_routes.routes[index];
409 g_routes.routes[index] = NULL;
419 int vnet_delete_route_entry_by_addr(char src_mac[6],
426 link_type_t src_type)
428 int index = find_route_entry(src_mac, dest_mac, src_mac_qual,
429 dest_mac_qual, link_idx, type, src, src_type);
435 return delete_route_entry(index);
438 static int match_route(uint8_t *src_mac, uint8_t *dst_mac, link_type_t src_type, int src_index, int *matches)
440 int values[MAX_ROUTES];
441 int matched_routes[MAX_ROUTES];
449 for(i = 0; i<MAX_ROUTES; i++) {
450 if (g_routes.routes[i] != NULL){
451 if ((g_routes.routes[i]->src_type != LINK_ANY) &&
452 ((g_routes.routes[i]->src_type != src_type) ||
453 ((g_routes.routes[i]->src_link_idx != src_index) &&
454 (g_routes.routes[i]->src_link_idx != -1)))) {
455 PrintDebug("Vnet: MatchRoute: Source route is on and does not match\n");
459 if ((g_routes.routes[i]->dest_mac_qual == MAC_ANY) &&
460 (g_routes.routes[i]->src_mac_qual == MAC_ANY)) {
461 matched_routes[num_matches] = i;
462 values[num_matches] = 3;
466 if (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) {
467 if (g_routes.routes[i]->src_mac_qual != MAC_NOT) {
468 if (g_routes.routes[i]->dest_mac_qual == MAC_ANY) {
469 matched_routes[num_matches] = i;
470 values[num_matches] = 6;
473 } else if (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) == 0) {
474 if (g_routes.routes[i]->dest_mac_qual != MAC_NOT) {
475 matched_routes[num_matches] = i;
476 values[num_matches] = 8;
484 if (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) == 0) {
485 if (g_routes.routes[i]->dest_mac_qual != MAC_NOT) {
486 if (g_routes.routes[i]->src_mac_qual == MAC_ANY) {
487 matched_routes[num_matches] = i;
488 values[num_matches] = 6;
491 } else if (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) {
492 if (g_routes.routes[i]->src_mac_qual != MAC_NOT) {
493 if (exact_match == 0) {
494 matched_routes[num_matches] = i;
495 values[num_matches] = 8;
503 if ((g_routes.routes[i]->dest_mac_qual == MAC_NOT) &&
504 (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) != 0)) {
505 if (g_routes.routes[i]->src_mac_qual == MAC_ANY) {
506 matched_routes[num_matches] = i;
507 values[num_matches] = 5;
509 } else if (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) {
510 if (g_routes.routes[i]->src_mac_qual != MAC_NOT) {
511 matched_routes[num_matches] = i;
512 values[num_matches] = 7;
518 if ((g_routes.routes[i]->src_mac_qual == MAC_NOT) &&
519 (memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) != 0)) {
520 if (g_routes.routes[i]->dest_mac_qual == MAC_ANY) {
521 matched_routes[num_matches] = i;
522 values[num_matches] = 5;
524 } else if (memcmp((void *)&g_routes.routes[i]->dest_mac, (void *)dst_mac, 6) == 0) {
525 if (g_routes.routes[i]->dest_mac_qual != MAC_NOT) {
526 matched_routes[num_matches] = i;
527 values[num_matches] = 7;
535 for(i = 0; i<MAX_ROUTES; i++) {
536 if ((memcmp((void *)&g_routes.routes[i]->src_mac, (void *)src_mac, 6) == 0) &&
537 (g_routes.routes[i]->dest_mac_qual == MAC_NONE) &&
538 ((g_routes.routes[i]->src_type == LINK_ANY) ||
539 ((g_routes.routes[i]->src_type == src_type) &&
540 ((g_routes.routes[i]->src_link_idx == src_index) ||
541 (g_routes.routes[i]->src_link_idx == -1))))) {
542 matched_routes[num_matches] = i;
543 values[num_matches] = 4;
544 PrintDebug("Vnet: MatchRoute: We matched a default route (%d)\n", i);
549 //If many rules have been matched, we choose one which has the highest value rating
550 if (num_matches == 0) {
554 for (i = 0; i < num_matches; i++) {
555 if (values[i] > max) {
558 matches[no] = matched_routes[i];
560 } else if (values[i] == max) {
561 matches[no] = matched_routes[i];
570 // TODO: To be removed
571 static int process_udpdata()
573 struct ethernet_pkt * pt;
576 uint16_t remote_port = 0;
577 SOCK link_sock = g_udp_sockfd;
578 int length = sizeof(struct ethernet_pkt) - (2 * sizeof(int)); //minus the "size" and "type"
580 //run in a loop to get packets from outside network, adding them to the incoming packet queue
582 pt = (struct ethernet_pkt *)V3_Malloc(sizeof(struct ethernet_pkt));
585 PrintError("Vnet: process_udp: Malloc fails\n");
589 PrintDebug("Vnet: route_thread: socket: [%d]. ready to receive from ip [%x], port [%d] or from VMs\n", link_sock, (uint_t)dest, remote_port);
590 pt->size = V3_RecvFrom_IP( link_sock, dest, remote_port, pt->data, length);
591 PrintDebug("Vnet: route_thread: socket: [%d] receive from ip [%x], port [%d]\n", link_sock, (uint_t)dest, remote_port);
594 PrintDebug("Vnet: process_udp: receiving packet from UDP fails\n");
599 PrintDebug("Vnet: process_udp: get packet\n");
600 print_packet(pt->data, pt->size);
604 // TODO: To be removed
605 static int indata_handler( )
613 // TODO: To be removed
614 static int start_recv_data()
619 SOCK udp_data_socket;
621 if ((udp_data_socket = V3_Create_UDP_Socket()) < 0){
622 PrintError("VNET: Can't setup udp socket\n");
625 PrintDebug("Vnet: vnet_setup_udp: get socket: %d\n", udp_data_socket);
626 g_udp_sockfd = udp_data_socket;
628 store_topologies(udp_data_socket);
630 if (V3_Bind_Socket(udp_data_socket, vnet_udp_port) < 0){
631 PrintError("VNET: Can't bind socket\n");
634 PrintDebug("VNET: vnet_setup_udp: bind socket successful\n");
637 V3_CREATE_THREAD(&indata_handler, NULL, "VNET_DATA_HANDLER");
641 static void store_test_topologies(SOCK fd)
644 int src_mac_qual = MAC_ANY;
645 int dest_mac_qual = MAC_ANY;
648 dest = (0 | 172 << 24 | 23 << 16 | 1 );
649 PrintDebug("VNET: store_topologies. NOT VNET_SERVER, dest = %x\n", dest);
651 dest = (0 | 172 << 24 | 23 << 16 | 2 );
652 PrintDebug("VNET: store_topologies. VNET_SERVER, dest = %x\n", dest);
657 int src_type= LINK_ANY; //ANY_SRC_TYPE
663 static int handle_one_pkt(struct ethernet_pkt * pkt)
665 int src_link_index = -1; //the value of src_link_index of udp always is 0
670 int matches[MAX_ROUTES];
671 int num_matched_routes = 0;
673 struct in_pkt_header header;
675 // get the ethernet and ip headers from the packet
676 memcpy((void *)&header, (void *)pkt->data, sizeof(header));
677 memcpy(src_mac, header.ethernetsrc, 6);
678 memcpy(dst_mac, header.ethernetdest, 6);
680 #ifdef CONFIG_DEBUG_VNET
684 mac_to_string(src_mac, src_str);
685 mac_to_string(dst_mac, dest_str);
687 PrintDebug("Vnet: HandleDataOverLink. SRC(%s), DEST(%s)\n", src_str, dest_str);
690 char hash_key[hash_key_size];
691 make_hash_key(hash_key, src_mac, dst_mac, LINK_EDGE, src_link_index);//link_edge -> pt->type???
693 num_matched_routes = look_into_cache((route_hashkey_t)hash_key, matches);
695 if (num_matched_routes == -1) { //no match
696 num_matched_routes = match_route(src_mac, dst_mac, LINK_ANY, src_link_index, matches);
698 if (num_matched_routes > 0) {
699 add_route_to_cache(hash_key, num_matched_routes,matches);
703 PrintDebug("Vnet: HandleDataOverLink: Matches=%d\n", num_matched_routes);
705 for (i = 0; i < num_matched_routes; i++) {
706 int route_index = -1;
710 route_index = matches[i];
712 PrintDebug("Vnet: HandleDataOverLink: Forward packet from link according to Route entry %d\n", route_index);
714 link_index = g_routes.routes[route_index]->link_idx;
715 if (link_index < 0 || link_index > MAX_LINKS)
718 struct link_entry *link = g_links.links[link_index];
723 if (g_routes.routes[route_index]->link_type == LINK_EDGE) {
725 // TODO: apply the header in the beginning of the packet to be sent
726 if ((link->dst_link->input(pkt->data, pkt_len, NULL)) != pkt_len)
728 } else if (g_routes.routes[route_index]->link_type == LINK_INTERFACE) {
731 if ((link->dst_link->input(pkt->data, pkt_len, NULL)) != pkt_len)
734 PrintError("Vnet: Wrong Edge type\n");
742 static int send_ethernet_pkt(char *buf, int length)
744 struct ethernet_pkt *pt;
746 pt = (struct ethernet_pkt *)V3_Malloc(sizeof(struct ethernet_pkt));
747 ethernet_packet_init(pt, buf, length); //====here we copy sending data once
749 PrintDebug("VNET: vm_send_pkt: transmitting packet: (size:%d)\n", (int)pt->size);
750 print_packet((char *)buf, length);
752 v3_enqueue(g_inpkt_q, (addr_t)pt);
757 int v3_vnet_send_pkt(uchar_t *buf, int length)
759 PrintDebug("VNET: In V3_Send_pkt: pkt length %d\n", length);
761 return send_ethernet_pkt((char *)buf, length);
764 static int search_device(char *device_name)
768 for (i = 0; i < MAX_LINKS; i++) {
769 if (g_links.links[i] != NULL && g_links.links[i]->type == LINK_INTERFACE) {
770 if (!strcmp(device_name, g_links.links[i]->dst_dev->name)) {
779 int vnet_register_device(struct vm_device *vdev,
782 int (*netif_input)(uchar_t * pkt, uint_t size, void *private_data),
785 struct vnet_if_device *if_dev;
787 int idx = search_device(dev_name);
791 if_dev = (struct vnet_if_device *)V3_Malloc(sizeof(struct vnet_if_device));
794 PrintError("VNET: Malloc fails\n");
798 strcpy(if_dev->name, dev_name);
799 strncpy(if_dev->mac_addr, mac, 6);
801 if_dev->input = netif_input;
802 if_dev->private_data = data;
804 struct link_entry *link = (struct link_entry *)V3_Malloc(sizeof(struct link_entry));
806 link->type = LINK_INTERFACE;
807 link->dst_dev = if_dev;
809 idx = add_link_entry(link);
814 int vnet_unregister_device(char *dev_name)
818 idx = search_device(dev_name);
824 struct vnet_if_device *device = (struct vnet_if_device *)delete_link_entry(idx);
825 if (device == NULL) {
834 int v3_vnet_pkt_process()
836 struct ethernet_pkt *pt;
838 PrintDebug("VNET: In vnet_check\n");
840 while ((pt = (struct ethernet_pkt *)v3_dequeue(g_inpkt_q)) != NULL) {
841 PrintDebug("VNET: In vnet_check: pt length %d, pt type %d\n", (int)pt->size, (int)pt->type);
842 v3_hexdump(pt->data, pt->size, NULL, 0);
844 if(handle_one_pkt(pt)) {
845 PrintDebug("VNET: vnet_check: handle one packet!\n");
847 PrintError("VNET: vnet_check: fail to forward one packet, discard it!\n");
850 V3_Free(pt); //be careful here
857 static void init_empty_link_table()
861 for (i = 0; i < MAX_LINKS; i++)
862 g_links.links[i] = NULL;
868 static void init_empty_route_table()
872 for (i = 0; i < MAX_ROUTES; i++)
873 g_routes.routes[i] = NULL;
878 static void init_tables() {
879 init_empty_link_table();
880 init_empty_route_table();
884 static void init_pkt_queue()
886 PrintDebug("VNET Init package receiving queue\n");
888 g_inpkt_q = v3_create_queue();
889 v3_init_queue(g_inpkt_q);
894 static int init_routing_tables(struct routing_entry *route_tab, uint16_t size)
896 //struct routing_entry *route;
904 static int init_link_tables(struct link_entry *link_tab, uint16_t size)
906 //struct link_entry *link;
913 struct table_init_info {
914 addr_t routing_table_start;
915 uint16_t routing_table_size;
916 addr_t link_table_start;
917 uint16_t link_table_size;
920 static int handle_init_tables_hcall(struct guest_info * info, uint_t hcall_id, void *priv_data)
923 addr_t info_addr = (addr_t)info->vm_regs.rcx;
925 if (guest_va_to_host_va(info, info_addr, (addr_t *)&(buf)) == -1) {
926 PrintError("Could not translate buffer address\n");
930 //struct table_init_info *init_info = (struct table_init_info *)buf;
937 void v3_vnet_init(struct guest_info *vm)
942 v3_register_hypercall(vm, VNET_INITAB_HCALL, handle_init_tables_hcall, NULL);
944 //store_test_topologies(udp_data_socket);
946 PrintDebug("VNET Initied\n");