-/*
- * This file is part of the Palacios Virtual Machine Monitor developed
- * by the V3VEE Project with funding from the United States National
- * Science Foundation and the Department of Energy.
- *
- * The V3VEE Project is a joint project between Northwestern University
- * and the University of New Mexico. You can find out more at
- * http://www.v3vee.org
- *
- * Copyright (c) 2011, Lei Xia <lxia@northwestern.edu>
- * Copyright (c) 2011, The V3VEE Project <http://www.v3vee.org>
- * All rights reserved.
- *
- * This is free software. You are permitted to use, redistribute,
- * and modify it under the terms of the GNU General Public License
- * Version 2 (GPLv2). The accompanying COPYING file contains the
- * full text of the license.
- */
-/* Palacios VNET Host Bridge */
+/*
+ * Palacios VNET Host Bridge
+ * (c) Lei Xia 2010
+ */
#include <linux/spinlock.h>
#include <linux/seq_file.h>
#include <linux/sched.h>
#include <asm/msr.h>
+#include <linux/net.h>
+#include <linux/socket.h>
+#include <net/sock.h>
+
#include <vnet/vnet.h>
#include <vnet/vnet_hashtable.h>
#include "palacios-vnet.h"
+#include "palacios.h"
+
+
#define VNET_SERVER_PORT 9000
+#define VNET_ADAPTIVE_BRIDGE 1 // set this to one to have bridge go to sleep if there nothing to do...
+#define VNET_NOPROGRESS_LIMIT 1000 // ... after this many iterations
+#define VNET_YIELD_TIME_USEC 1000 // ... and go to sleep for this long
+
struct vnet_link {
uint32_t dst_ip;
uint16_t dst_port;
static void _delete_link(struct vnet_link * link){
- unsigned long flags;
+ unsigned long flags = 0;
link->sock->ops->release(link->sock);
- spin_lock_irqsave(&(vnet_brg_s.lock), flags);
+ palacios_spinlock_lock_irqsave(&(vnet_brg_s.lock), flags);
list_del(&(link->node));
vnet_htable_remove(vnet_brg_s.ip2link, (addr_t)&(link->dst_ip), 0);
vnet_brg_s.num_links --;
- spin_unlock_irqrestore(&(vnet_brg_s.lock), flags);
+ palacios_spinlock_unlock_irqrestore(&(vnet_brg_s.lock), flags);
- printk("VNET Bridge: Link deleted, ip 0x%x, port: %d, idx: %d\n",
+ INFO("VNET Bridge: Link deleted, ip 0x%x, port: %d, idx: %d\n",
link->dst_ip,
link->dst_port,
link->idx);
- kfree(link);
+ palacios_free(link);
link = NULL;
}
}
static void deinit_links_list(void){
- struct vnet_link * link;
+ struct vnet_link * link = NULL, * tmp_link = NULL;
- list_for_each_entry(link, &(vnet_brg_s.link_list), node) {
+ list_for_each_entry_safe(link, tmp_link, &(vnet_brg_s.link_list), node) {
_delete_link(link);
}
}
break;
default:
- printk("Unsupported VNET Server Protocol\n");
+ WARNING("Unsupported VNET Server Protocol\n");
return -1;
}
if ((err = sock_create(AF_INET, SOCK_DGRAM, protocol, &link->sock)) < 0) {
- printk("Could not create socket for VNET Link, error %d\n", err);
+ WARNING("Could not create socket for VNET Link, error %d\n", err);
return -1;
}
+ if (link->sock_proto == UDP) {
+ // no UDP checksumming
+ lock_sock(link->sock->sk);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0)
+ link->sock->sk->sk_no_check = 1;
+#else
+ link->sock->sk->sk_no_check_tx = 1;
+ link->sock->sk->sk_no_check_rx = 1;
+#endif
+ release_sock(link->sock->sk);
+ }
+
memset(&link->sock_addr, 0, sizeof(struct sockaddr));
link->sock_addr.sin_family = AF_INET;
link->sock_addr.sin_addr.s_addr = link->dst_ip;
link->sock_addr.sin_port = htons(link->dst_port);
- if ((err = link->sock->ops->connect(link->sock, (struct sockaddr *)&(link->sock_addr), sizeof(struct sockaddr), 0) < 0)) {
- printk("Could not connect to remote VNET Server, error %d\n", err);
+
+ if ((err = link->sock->ops->connect(link->sock, (struct sockaddr *)&(link->sock_addr), sizeof(struct sockaddr), 0)) < 0) {
+ WARNING("Could not connect to remote VNET Server, error %d\n", err);
return -1;
}
- spin_lock_irqsave(&(vnet_brg_s.lock), flags);
+
+ palacios_spinlock_lock_irqsave(&(vnet_brg_s.lock), flags);
list_add(&(link->node), &(vnet_brg_s.link_list));
vnet_brg_s.num_links ++;
link->idx = ++ vnet_brg_s.link_idx;
vnet_htable_insert(vnet_brg_s.ip2link, (addr_t)&(link->dst_ip), (addr_t)link);
- spin_unlock_irqrestore(&(vnet_brg_s.lock), flags);
+ palacios_spinlock_unlock_irqrestore(&(vnet_brg_s.lock), flags);
- printk("VNET Bridge: Link created, ip 0x%x, port: %d, idx: %d, link: %p, protocol: %s\n",
+ INFO("VNET Bridge: Link created, ip 0x%x, port: %d, idx: %d, link: %p, protocol: %s\n",
link->dst_ip,
link->dst_port,
link->idx,
struct vnet_link * new_link = NULL;
uint32_t idx;
- new_link = kmalloc(sizeof(struct vnet_link), GFP_KERNEL);
+ new_link = palacios_alloc(sizeof(struct vnet_link));
if (!new_link) {
return -1;
}
idx = _create_link(new_link);
if (idx < 0) {
- printk("Could not create link\n");
- kfree(new_link);
+ WARNING("Could not create link\n");
+ palacios_free(new_link);
return -1;
}
iov.iov_base = buf;
iov.iov_len = len;
- msg.msg_flags = 0;
+ msg.msg_flags = MSG_NOSIGNAL;
msg.msg_name = addr;
msg.msg_namelen = sizeof(struct sockaddr_in);
msg.msg_control = NULL;
msg.msg_controllen = 0;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0)
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
+#else
+ iov_iter_init(&(msg.msg_iter),WRITE,&iov,1,0);
+#endif
msg.msg_control = NULL;
oldfs = get_fs();
static int
_udp_recv(struct socket * sock,
- struct sockaddr_in * addr,
- unsigned char * buf, int len) {
+ struct sockaddr_in * addr,
+ unsigned char * buf, int len, int nonblocking) {
struct msghdr msg;
struct iovec iov;
mm_segment_t oldfs;
iov.iov_base = buf;
iov.iov_len = len;
- msg.msg_flags = 0;
+ msg.msg_flags = MSG_NOSIGNAL | (nonblocking ? MSG_DONTWAIT : 0);
msg.msg_name = addr;
msg.msg_namelen = sizeof(struct sockaddr_in);
msg.msg_control = NULL;
msg.msg_controllen = 0;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,19,0)
msg.msg_iov = &iov;
msg.msg_iovlen = 1;
+#else
+ iov_iter_init(&(msg.msg_iter),READ,&iov,1,0);
+#endif
msg.msg_control = NULL;
oldfs = get_fs();
int len,
int link_id){
struct v3_vnet_pkt pkt;
+ memset(&pkt,0,sizeof(struct v3_vnet_pkt));
pkt.size = len;
+ pkt.dst_type = LINK_NOSET;
pkt.src_type = LINK_EDGE;
pkt.src_id = link_id;
memcpy(pkt.header, buf, ETHERNET_HEADER_LEN);
pkt.data = buf;
if(net_debug >= 2){
- printk("VNET Lnx Bridge: send pkt to VNET core (size: %d, src_id: %d, src_type: %d)\n",
+ DEBUG("VNET Lnx Bridge: send pkt to VNET core (size: %d, src_id: %d, src_type: %d)\n",
pkt.size, pkt.src_id, pkt.src_type);
if(net_debug >= 4){
print_hex_dump(NULL, "pkt_data: ", 0, 20, 20, pkt.data, pkt.size, 0);
vnet_brg_s.stats.pkt_to_vmm ++;
- return v3_vnet_send_pkt(&pkt, NULL, 1);
+ return v3_vnet_send_pkt(&pkt, NULL);
}
bridge_send_pkt(struct v3_vm_info * vm,
struct v3_vnet_pkt * pkt,
void * private_data) {
- struct vnet_link * link;
+ struct vnet_link * link = NULL;
if(net_debug >= 2){
- printk("VNET Lnx Host Bridge: packet received from VNET Core ... pkt size: %d, link: %d\n",
+ DEBUG("VNET Lnx Host Bridge: packet received from VNET Core ... pkt size: %d, link: %d\n",
pkt->size,
pkt->dst_id);
if(net_debug >= 4){
break;
default:
- printk("VNET Server: Invalid Link Protocol\n");
+ WARNING("VNET Server: Invalid Link Protocol\n");
vnet_brg_s.stats.pkt_drop_vmm ++;
}
link->stats.tx_bytes += pkt->size;
link->stats.tx_pkts ++;
} else {
- printk("VNET Bridge Linux Host: wrong dst link, idx: %d, discards the packet\n", pkt->dst_id);
+ INFO("VNET Bridge Linux Host: wrong dst link, idx: %d, discarding the packet\n", pkt->dst_id);
vnet_brg_s.stats.pkt_drop_vmm ++;
}
break;
default:
- printk("Unsupported VNET Server Protocol\n");
+ WARNING("Unsupported VNET Server Protocol\n");
return -1;
}
if ((err = sock_create(AF_INET, SOCK_DGRAM, protocol, &vnet_brg_s.serv_sock)) < 0) {
- printk("Could not create VNET server socket, error: %d\n", err);
+ WARNING("Could not create VNET server socket, error: %d\n", err);
return -1;
}
+ if (vnet_brg_s.serv_proto == UDP) {
+ // No UDP checksumming is done
+ lock_sock(vnet_brg_s.serv_sock->sk);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(3,16,0)
+ vnet_brg_s.serv_sock->sk->sk_no_check = 1;
+#else
+ vnet_brg_s.serv_sock->sk->sk_no_check_tx = 1;
+ vnet_brg_s.serv_sock->sk->sk_no_check_rx = 1;
+#endif
+ release_sock(vnet_brg_s.serv_sock->sk);
+ }
+
memset(&vnet_brg_s.serv_addr, 0, sizeof(struct sockaddr));
vnet_brg_s.serv_addr.sin_family = AF_INET;
vnet_brg_s.serv_addr.sin_port = htons(VNET_SERVER_PORT);
if ((err = vnet_brg_s.serv_sock->ops->bind(vnet_brg_s.serv_sock, (struct sockaddr *)&(vnet_brg_s.serv_addr), sizeof(struct sockaddr))) < 0) {
- printk("Could not bind VNET server socket to port %d, error: %d\n", VNET_SERVER_PORT, err);
+ WARNING("Could not bind VNET server socket to port %d, error: %d\n", VNET_SERVER_PORT, err);
return -1;
}
- printk("VNET server bind to port: %d\n", VNET_SERVER_PORT);
+ INFO("VNET server bind to port: %d\n", VNET_SERVER_PORT);
if(vnet_brg_s.serv_proto == TCP){
if((err = vnet_brg_s.serv_sock->ops->listen(vnet_brg_s.serv_sock, 32)) < 0){
- printk("VNET Server error listening on port %d, error %d\n", VNET_SERVER_PORT, err);
+ WARNING("VNET Server error listening on port %d, error %d\n", VNET_SERVER_PORT, err);
return -1;
}
}
+
+
return 0;
}
struct sockaddr_in pkt_addr;
struct vnet_link * link = NULL;
int len;
+ uint64_t noprogress_count;
+
+ INFO("Palacios VNET Bridge: UDP receiving server ..... \n");
+
+ pkt = palacios_alloc(MAX_PACKET_LEN);
+
+ if (!pkt) {
+ ERROR("Unable to allocate packet in VNET UDP Server\n");
+ return -1;
+ }
- printk("Palacios VNET Bridge: UDP receiving server ..... \n");
+
+ noprogress_count=0;
- pkt = kmalloc(MAX_PACKET_LEN, GFP_KERNEL);
while (!kthread_should_stop()) {
+
+ // This is a NONBLOCKING receive
+ // If we block here, we will never detect that this thread
+ // is being signaled to stop, plus we might go uninterrupted on this core
+ // blocking out access to other threads - leave this NONBLOCKING
+ // unless you know what you are doing
+ len = _udp_recv(vnet_brg_s.serv_sock, &pkt_addr, pkt, MAX_PACKET_LEN, 1);
+
+
+ // If it would have blocked, we have no packet, and so
+ // we will give other threads on this core a chance
+ if (len==-EAGAIN || len==-EWOULDBLOCK || len==-EINTR) {
+
+ // avoid rollover in the counter out of paranoia
+ if (! ((noprogress_count + 1) < noprogress_count)) {
+ noprogress_count++;
+ }
+
+ // adaptively select yielding strategy depending on
+ // whether we are making progress
+ if ((!VNET_ADAPTIVE_BRIDGE) || (noprogress_count < VNET_NOPROGRESS_LIMIT)) {
+ // Likely making progress, do fast yield so we
+ // come back immediately if there is no other action
+ palacios_yield_cpu();
+ } else {
+ // Likely not making progress, do potentially slow
+ // yield - we won't come back for until VNET_YIELD_TIME_USEC has passed
+ palacios_sleep_cpu(VNET_YIELD_TIME_USEC);
+ }
+
+ continue;
+ }
- len = _udp_recv(vnet_brg_s.serv_sock, &pkt_addr, pkt, MAX_PACKET_LEN);
+
+ // Something interesting has happened, therefore progress!
+ noprogress_count=0;
+
+
if(len < 0) {
- printk("Receive error: Could not get packet, error %d\n", len);
+ WARNING("Receive error: Could not get packet, error %d\n", len);
continue;
}
link = _link_by_ip(pkt_addr.sin_addr.s_addr);
+
if (link == NULL){
- printk("VNET Server: No VNET Link match the src IP\n");
+ WARNING("VNET Server: No VNET Link matches the src IP\n");
vnet_brg_s.stats.pkt_drop_phy ++;
continue;
}
send_to_palacios(pkt, len, link->idx);
}
- kfree(pkt);
+ INFO("VNET Server: UDP thread exiting\n");
+
+ palacios_free(pkt);
return 0;
}
//accept new connection
//use select to receive pkt from physical network
//or create new kthread to handle each connection?
+ WARNING("VNET Server: TCP is not currently supported\n");
+ return -1;
}else {
- printk ("VNET Server: Unsupported Protocol\n");
+ WARNING ("VNET Server: Unsupported Protocol\n");
return -1;
}
memset(&vnet_brg_s, 0, sizeof(struct vnet_brg_state));
INIT_LIST_HEAD(&(vnet_brg_s.link_list));
- spin_lock_init(&(vnet_brg_s.lock));
+ palacios_spinlock_init(&(vnet_brg_s.lock));
vnet_brg_s.serv_proto = UDP;
vnet_brg_s.ip2link = vnet_create_htable(10, hash_fn, hash_eq);
if(vnet_brg_s.ip2link == NULL){
- printk("Failure to initiate VNET link hashtable\n");
+ WARNING("Failure to initiate VNET link hashtable\n");
return -1;
}
if(init_vnet_serv() < 0){
- printk("Failure to initiate VNET server\n");
+ WARNING("Failure to initiate VNET server\n");
return -1;
}
- vnet_brg_s.serv_thread = kthread_run(_rx_server, NULL, "vnet-server");
+ vnet_brg_s.serv_thread = kthread_run(_rx_server, NULL, "vnet_brgd");
bridge_ops.input = bridge_send_pkt;
bridge_ops.poll = NULL;
if( v3_vnet_add_bridge(NULL, &bridge_ops, HOST_LNX_BRIDGE, NULL) < 0){
- printk("VNET LNX Bridge: Fails to register bridge to VNET core");
+ WARNING("VNET LNX Bridge: Fails to register bridge to VNET core");
}
- printk("VNET Linux Bridge initiated\n");
+ INFO("VNET Linux Bridge initiated\n");
return 0;
}
void vnet_bridge_deinit(void){
+ INFO("VNET LNX Bridge Deinit Started\n");
+
v3_vnet_del_bridge(HOST_LNX_BRIDGE);
+ //DEBUG("Stopping bridge service thread\n");
+
kthread_stop(vnet_brg_s.serv_thread);
+
+ //DEBUG("Releasing bridee service socket\n");
+
vnet_brg_s.serv_sock->ops->release(vnet_brg_s.serv_sock);
+ //DEBUG("Deiniting bridge links\n");
+
deinit_links_list();
+ //DEBUG("Freeing bridge hash tables\n");
+
vnet_free_htable(vnet_brg_s.ip2link, 0, 0);
vnet_brg_s.status = 0;
+
+ palacios_spinlock_deinit(&(vnet_brg_s.lock));
+
+ INFO("VNET LNX Bridge Deinit Finished\n");
}
