From: Peter Dinda <pdinda@northwestern.edu>
Date: Thu, 9 Aug 2012 18:50:02 +0000 (-0500)
Subject: Configurable yielding strategies in VNET bridge thread and tx kick threads
X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=b5345655d398d9585826779065093d29d23dfb30

Configurable yielding strategies in VNET bridge thread and tx kick threads
---

diff --git a/linux_module/palacios-vnet-brg.c b/linux_module/palacios-vnet-brg.c
index 309171d..999adcd 100644
--- a/linux_module/palacios-vnet-brg.c
+++ b/linux_module/palacios-vnet-brg.c
@@ -32,9 +32,9 @@
 
 #define VNET_SERVER_PORT 9000
 
-#define VNET_NOPROGRESS_LIMIT 1000
-
-#define VNET_YIELD_TIME_USEC  1000
+#define VNET_ADAPTIVE_BRIDGE  0      // set this to one to have bridge go to sleep if there nothing to do...
+#define VNET_NOPROGRESS_LIMIT 1000   // ... after this many iterations
+#define VNET_YIELD_TIME_USEC  1000   // ... and go to sleep for this long
 
 struct vnet_link {
     uint32_t dst_ip;
@@ -472,7 +472,7 @@ static int _udp_server(void * arg) {
 	    
 	    // adaptively select yielding strategy depending on
 	    // whether we are making progress
-	    if (noprogress_count < VNET_NOPROGRESS_LIMIT) { 
+	    if ((!VNET_ADAPTIVE_BRIDGE) || (noprogress_count < VNET_NOPROGRESS_LIMIT)) { 
 		// Likely making progress, do fast yield so we 
 		// come back immediately if there is no other action
 		palacios_yield_cpu();
diff --git a/palacios/src/vnet/vnet_core.c b/palacios/src/vnet/vnet_core.c
index 0819adf..7613368 100644
--- a/palacios/src/vnet/vnet_core.c
+++ b/palacios/src/vnet/vnet_core.c
@@ -31,8 +31,12 @@
 #define PrintDebug(fmt, args...)
 #endif
 
-#define VNET_NOPROGRESS_LIMIT 1000
-#define VNET_YIELD_USEC       1000
+#define VNET_NUM_TX_KICK_THREADS 2
+
+#define VNET_ADAPTIVE_TX_KICK 0         // set to 1 to try to sleep when there is nothing to do
+#define VNET_NOPROGRESS_LIMIT 1000      //   ... after this many tries
+#define VNET_YIELD_USEC       1000      //   ... and go to sleep for this long
+
 
 int net_debug = 0;
 
@@ -119,9 +123,10 @@ static struct {
    /* device queue that are waiting to be polled */
     struct v3_queue * poll_devs;
 
-    struct vnet_thread * pkt_flush_thread;
+    struct vnet_thread * pkt_flush_thread[VNET_NUM_TX_KICK_THREADS];
 
     struct hashtable * route_cache;
+
 } vnet_state;
 	
 
@@ -607,15 +612,15 @@ int v3_vnet_query_header(uint8_t src_mac[ETH_ALEN],
 
     p.size=14;
     p.data=p.header;
-    memcpy(p.header,dest_mac,6);
-    memcpy(p.header+6,src_mac,6);
+    memcpy(p.header,dest_mac,ETH_ALEN);
+    memcpy(p.header+ETH_ALEN,src_mac,ETH_ALEN);
     memset(p.header+12,0,2);
 
     p.src_type = LINK_EDGE;
     p.src_id = 0;
 
-    memcpy(header->src_mac,src_mac,6);
-    memcpy(header->dst_mac,dest_mac,6);
+    memcpy(header->src_mac,src_mac,ETH_ALEN);
+    memcpy(header->dst_mac,dest_mac,ETH_ALEN);
 
 
     flags = vnet_lock_irqsave(vnet_state.lock);
@@ -785,7 +790,7 @@ int v3_vnet_add_dev(struct v3_vm_info * vm, uint8_t * mac,
 	return -1;
     }
    
-    memcpy(new_dev->mac_addr, mac, 6);
+    memcpy(new_dev->mac_addr, mac, ETH_ALEN);
     new_dev->dev_ops.input = ops->input;
     new_dev->dev_ops.poll = ops->poll;
     new_dev->private_data = priv_data;
@@ -961,8 +966,8 @@ static int vnet_tx_flush(void * args){
 	return -1;
     }
 
-    noprogress_count=0;
-
+    noprogress_count = 0;
+    
     while (!vnet_thread_should_stop()) {
 
 	more=0; // will indicate if any device has more work for us to do
@@ -998,7 +1003,7 @@ static int vnet_tx_flush(void * args){
 	}
 
 	// adaptively yield 
-	if (noprogress_count < VNET_NOPROGRESS_LIMIT) { 
+	if ((!VNET_ADAPTIVE_TX_KICK) || (noprogress_count < VNET_NOPROGRESS_LIMIT)) { 
 	    V3_Yield();
 	} else {
 	    V3_Yield_Timed(VNET_YIELD_USEC);
@@ -1013,7 +1018,10 @@ static int vnet_tx_flush(void * args){
     return 0;
 }
 
-int v3_init_vnet() {
+int v3_init_vnet() 
+{
+    int i;
+
     memset(&vnet_state, 0, sizeof(vnet_state));
 	
     INIT_LIST_HEAD(&(vnet_state.routes));
@@ -1034,22 +1042,31 @@ int v3_init_vnet() {
 
     vnet_state.poll_devs = v3_create_queue();
 
-    vnet_state.pkt_flush_thread = vnet_start_thread(vnet_tx_flush, NULL, "vnetd-1");
+    for (i=0; i<VNET_NUM_TX_KICK_THREADS;i++) { 
+	char name[32];
+	snprintf(name,32,"vnetd-%d",i);
+	vnet_state.pkt_flush_thread[i] = vnet_start_thread(vnet_tx_flush, NULL, name);
+    }
 
-    PrintDebug("VNET/P is initiated\n");
+    PrintDebug("VNET/P is initiated (%d tx kick threads active)\n",VNET_NUM_TX_KICK_THREADS);
 
     return 0;
 }
 
 
-void v3_deinit_vnet() {
+void v3_deinit_vnet() 
+{
+    int i;
 
     v3_deinit_queue(vnet_state.poll_devs);
     Vnet_Free(vnet_state.poll_devs);
 
-    PrintDebug("Stopping flush thread\n");
-    // This will pause until the flush thread is gone
-    vnet_thread_stop(vnet_state.pkt_flush_thread);
+    for (i=0; i<VNET_NUM_TX_KICK_THREADS;i++) { 
+	PrintDebug("Stopping tx kick thread %d\n",i);
+	// This will pause until the flush thread is gone
+	vnet_thread_stop(vnet_state.pkt_flush_thread[i]);
+    }
+
     // At this point there should be no lock-holder
 
     Vnet_Free(vnet_state.poll_devs);