Palacios Public Git Repository

To checkout Palacios execute

  git clone http://v3vee.org/palacios/palacios.web/palacios.git
This will give you the master branch. You probably want the devel branch or one of the release branches. To switch to the devel branch, simply execute
  cd palacios
  git checkout --track -b devel origin/devel
The other branches are similar.


Context-based output infrastructure (V3_Print, etc) and modifications to use it
[palacios.git] / palacios / src / vnet / vnet_core.c
index c8ce677..7d9942d 100644 (file)
 #define PrintDebug(fmt, args...)
 #endif
 
-#define VNET_YIELD_USEC 1000
+#define VNET_NUM_TX_KICK_THREADS 1
+
+#define VNET_ADAPTIVE_TX_KICK 0         // set to 1 to try to sleep when there is nothing to do
+#define VNET_NOPROGRESS_LIMIT 1000      //   ... after this many tries
+#define VNET_YIELD_USEC       1000      //   ... and go to sleep for this long
+
 
 int net_debug = 0;
 
@@ -118,9 +123,10 @@ static struct {
    /* device queue that are waiting to be polled */
     struct v3_queue * poll_devs;
 
-    struct vnet_thread * pkt_flush_thread;
+    struct vnet_thread * pkt_flush_thread[VNET_NUM_TX_KICK_THREADS];
 
     struct hashtable * route_cache;
+
 } vnet_state;
        
 
@@ -135,15 +141,15 @@ static void print_route(struct v3_vnet_route * route){
     char str[50];
 
     mac2str(route->src_mac, str);
-    PrintDebug("Src Mac (%s),  src_qual (%d)\n", 
+    PrintDebug(VM_NONE, VCORE_NONE, "Src Mac (%s),  src_qual (%d)\n", 
               str, route->src_mac_qual);
     mac2str(route->dst_mac, str);
-    PrintDebug("Dst Mac (%s),  dst_qual (%d)\n", 
+    PrintDebug(VM_NONE, VCORE_NONE, "Dst Mac (%s),  dst_qual (%d)\n", 
               str, route->dst_mac_qual);
-    PrintDebug("Src dev id (%d), src type (%d)", 
+    PrintDebug(VM_NONE, VCORE_NONE, "Src dev id (%d), src type (%d)", 
               route->src_id, 
               route->src_type);
-    PrintDebug("Dst dev id (%d), dst type (%d)\n", 
+    PrintDebug(VM_NONE, VCORE_NONE, "Dst dev id (%d), dst type (%d)\n", 
               route->dst_id, 
               route->dst_type);
 }
@@ -151,13 +157,13 @@ static void print_route(struct v3_vnet_route * route){
 static void dump_routes(){
     struct vnet_route_info *route;
 
-    PrintDebug("\n========Dump routes starts ============\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "\n========Dump routes starts ============\n");
     list_for_each_entry(route, &(vnet_state.routes), node) {
-       PrintDebug("\nroute %d:\n", route->idx);
+       PrintDebug(VM_NONE, VCORE_NONE, "\nroute %d:\n", route->idx);
                
        print_route(&(route->route_def));
        if (route->route_def.dst_type == LINK_INTERFACE) {
-           PrintDebug("dst_dev (%p), dst_dev_id (%d), dst_dev_ops(%p), dst_dev_data (%p)\n",
+           PrintDebug(VM_NONE, VCORE_NONE, "dst_dev (%p), dst_dev_id (%d), dst_dev_ops(%p), dst_dev_data (%p)\n",
                route->dst_dev,
                route->dst_dev->dev_id,
                (void *)&(route->dst_dev->dev_ops),
@@ -165,7 +171,7 @@ static void dump_routes(){
        }
     }
 
-    PrintDebug("\n========Dump routes end ============\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "\n========Dump routes end ============\n");
 }
 
 #endif
@@ -189,7 +195,7 @@ static int add_route_to_cache(const struct v3_vnet_pkt * pkt, struct route_list
     memcpy(routes->hash_buf, pkt->hash_buf, VNET_HASH_SIZE);    
 
     if (vnet_htable_insert(vnet_state.route_cache, (addr_t)routes->hash_buf, (addr_t)routes) == 0) {
-       PrintError("VNET/P Core: Failed to insert new route entry to the cache\n");
+       PrintError(VM_NONE, VCORE_NONE, "VNET/P Core: Failed to insert new route entry to the cache\n");
        return -1;
     }
     
@@ -236,6 +242,10 @@ static struct vnet_dev * dev_by_mac(uint8_t * mac) {
 }
 
 
+static int start_vnet_kick_threads(void);
+static int stop_vnet_kick_threads(void);
+
+
 int v3_vnet_find_dev(uint8_t  * mac) {
     struct vnet_dev * dev = NULL;
 
@@ -256,14 +266,14 @@ int v3_vnet_add_route(struct v3_vnet_route route) {
     new_route = (struct vnet_route_info *)Vnet_Malloc(sizeof(struct vnet_route_info));
 
     if (!new_route) {
-       PrintError("Cannot allocate new route\n");
+       PrintError(VM_NONE, VCORE_NONE, "Cannot allocate new route\n");
        return -1;
     }
 
     memset(new_route, 0, sizeof(struct vnet_route_info));
 
 #ifdef V3_CONFIG_DEBUG_VNET
-    PrintDebug("VNET/P Core: add_route_entry:\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: add_route_entry:\n");
     print_route(&route);
 #endif
     
@@ -345,175 +355,287 @@ static void inline del_routes_by_dev(int dev_id){
            Vnet_Free(route);    
        }
     }
-
+    
     vnet_unlock_irqrestore(vnet_state.lock, flags);
 }
 
 
+// Match classes, must be in order
+#define NUM_MATCH_CLASSES 4
+#define NUM_MATCH_CLASSES_BOUND 3
+#define NONE    0
+#define NOT     1
+#define ANY     2
+#define DIRECT  3
 
 
-/* At the end allocate a route_list
- * This list will be inserted into the cache so we don't need to free it
- */
-static struct route_list * match_route(const struct v3_vnet_pkt * pkt) {
+static inline uint8_t match_mac(uint8_t test_mac[ETH_ALEN], 
+                               uint8_t route_mac[ETH_ALEN], 
+                               uint8_t route_qual)
+{
+    switch (route_qual) { 
+       case MAC_NOSET:
+           return NONE;
+           break;
+       case MAC_NONE:
+           return NONE;
+           break;
+       case MAC_ANY:
+           return ANY;
+           break;
+       case MAC_NOT:
+           if (memcmp(test_mac,route_mac,ETH_ALEN)) { 
+               return NOT;
+           } else {
+               return NONE;
+           }
+           break;
+       case MAC_ADDR:
+           if (memcmp(test_mac,route_mac,ETH_ALEN)) { 
+               return NONE;
+           } else {
+               return DIRECT;
+           }
+           break;
+       default:
+           PrintError(VM_NONE, VCORE_NONE, "Unknown qualifier %u\n",route_qual);
+           return NONE;
+           break;
+    }
+
+}
+
+#define QUAL_TO_STR(q)  (       \
+(q)==MAC_NOSET ? "MAC_NOSET" :  \
+(q)==MAC_NONE ? "MAC_NONE" :    \
+(q)==MAC_ANY ? "MAC_ANY" :      \
+(q)==MAC_NOT ? "MAC_NOT" :      \
+(q)==MAC_ADDR ? "MAC_ADDR" :    \
+"***UNDEFINED****"              \
+    )                           \
+
+#define MATCH_CLASS_TO_STR(c)  (       \
+(c)==NONE ? "NONE" :  \
+(c)==NOT ? "NOT" :    \
+(c)==ANY ? "ANY" :      \
+(c)==DIRECT ? "DIRECT" :      \
+"***UNDEFINED****"              \
+    )                           \
+
+
+
+/*
+
+Original priority behavior... 
+  
+priority   src  srcqual   dst  dstqual
+3              ANY            ANY
+4        X                    NONE
+5              ANY     X      NOT
+5        X     NOT            ANY
+6        X     ~NOT           ANY
+6              ANY     X      ~NOT
+7        X     ~NOT    X      NOT
+7        X     NOT     X      ~NOT
+8        X     ~NOT    X      ~NOT
+8        X     ~NOT    X      ~NOT
+
+*/
+
+/*
+  Current priority order is given in the following table
+*/
+
+// [src][dst] => priority
+static int priority_map[NUM_MATCH_CLASSES][NUM_MATCH_CLASSES] = 
+{
+    [NONE] = { [ 0 ... NUM_MATCH_CLASSES_BOUND ] = -1},   // ignore if it's not a source match
+    [NOT][NONE]                          = -1,            // ignore it if there is no destination match   
+    [NOT][NOT]                           = 3,                                   
+    [NOT][ANY]                           = 5,
+    [NOT][DIRECT]                        = 7,
+    [ANY][NONE]                          = -1,            // ignore if there is no destination match
+    [ANY][NOT]                           = 5,
+    [ANY][ANY]                           = 6,
+    [ANY][DIRECT]                        = 6,
+    [DIRECT][NONE]                       = -1,            // ignore if there is no destination match
+    [DIRECT][NOT]                        = 7,            
+    [DIRECT][ANY]                        = 8,            
+    [DIRECT][DIRECT]                     = 8,            
+};
+
+
+
+
+static inline int match_priority(uint8_t src_mac[ETH_ALEN],
+                                uint8_t dst_mac[ETH_ALEN],
+                                uint8_t route_src_mac[ETH_ALEN],
+                                uint8_t route_src_qual,
+                                uint8_t route_dst_mac[ETH_ALEN],
+                                uint8_t route_dst_qual)
+
+{
+
+    return priority_map[match_mac(src_mac,route_src_mac,route_src_qual)][match_mac(dst_mac,route_dst_mac,route_dst_qual)];
+}
+
+
+/*
+  Route matching will return the list of the highest priority routes that
+  match.  It's a list because it's possible to have multiple high priority routes
+ */ 
+static struct route_list * match_route(const struct v3_vnet_pkt * pkt) 
+{
+    int i;
     struct vnet_route_info * route = NULL; 
     struct route_list * matches = NULL;
     int num_matches = 0;
-    int max_rank = 0;
+    int max_priority = -1;
     struct list_head match_list;
     struct eth_hdr * hdr = (struct eth_hdr *)(pkt->data);
-    //  uint8_t src_type = pkt->src_type;
-    //  uint32_t src_link = pkt->src_id;
 
+    //
+    //
+    // NOTE: USING THE MATCH_NODE in the route list to record a match list
+    // IS A DISASTER WAITING TO HAPPEN
+    //
+    
 #ifdef V3_CONFIG_DEBUG_VNET
     {
-       char dst_str[100];
-       char src_str[100];
-
+       char dst_str[32], src_str[32];
        mac2str(hdr->src_mac, src_str);  
        mac2str(hdr->dst_mac, dst_str);
-       PrintDebug("VNET/P Core: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
+       PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: match_route. pkt: SRC(%s), DEST(%s)\n", src_str, dst_str);
     }
 #endif
-
-    INIT_LIST_HEAD(&match_list);
     
-#define UPDATE_MATCHES(rank) do {                              \
-       if (max_rank < (rank)) {                                \
-           max_rank = (rank);                                  \
-           INIT_LIST_HEAD(&match_list);                        \
-                                                               \
-           list_add(&(route->match_node), &match_list);        \
-           num_matches = 1;                                    \
-       } else if (max_rank == (rank)) {                        \
-           list_add(&(route->match_node), &match_list);        \
-           num_matches++;                                      \
-       }                                                       \
-    } while (0)
+    INIT_LIST_HEAD(&match_list);                       
+    
     
-
     list_for_each_entry(route, &(vnet_state.routes), node) {
+       
        struct v3_vnet_route * route_def = &(route->route_def);
+       
+       int priority;
+       
+       priority = match_priority(hdr->src_mac,
+                                 hdr->dst_mac,
+                                 route_def->src_mac,
+                                 route_def->src_mac_qual,
+                                 route_def->dst_mac,
+                                 route_def->dst_mac_qual);
 
-/*
-       // CHECK SOURCE TYPE HERE
-       if ( (route_def->src_type != LINK_ANY) && 
-            ( (route_def->src_type != src_type) || 
-              ( (route_def->src_id != src_link) &&
-                (route_def->src_id != -1)))) {
-           continue;
+       
+
+#ifdef V3_CONFIG_DEBUG_VNET
+       {
+           char dst_str[32];
+           char src_str[32];
+           
+           mac2str(route_def->src_mac, src_str);  
+           mac2str(route_def->dst_mac, dst_str);
+           
+           PrintDebug(VM_NONE, VCORE_NONE, "Tested match against SRC(%s) SRC_QUAL(%s), DEST(%s) DST_QUAL(%s): "
+                      "SRC_MATCH=%s  DEST_MATCH=%s PRIORITY=%d\n", 
+                      src_str, QUAL_TO_STR(route_def->src_mac_qual), 
+                      dst_str, QUAL_TO_STR(route_def->dst_mac_qual),
+                      MATCH_CLASS_TO_STR(match_mac(hdr->src_mac,route_def->src_mac,route_def->src_mac_qual)),
+                      MATCH_CLASS_TO_STR(match_mac(hdr->dst_mac,route_def->dst_mac,route_def->dst_mac_qual)),
+                  priority);
        }
-*/
+#endif
 
-       if ((route_def->dst_mac_qual == MAC_ANY) &&
-           (route_def->src_mac_qual == MAC_ANY)) {      
-           UPDATE_MATCHES(3);
+       if (priority<0) { 
+           PrintDebug(VM_NONE, VCORE_NONE, "No match to this rule\n");
+           continue;
        }
-       
-       if (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) {
-           if (route_def->src_mac_qual != MAC_NOT) {
-               if (route_def->dst_mac_qual == MAC_ANY) {
-                   UPDATE_MATCHES(6);
-               } else if (route_def->dst_mac_qual != MAC_NOT &&
-                          memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
-                   UPDATE_MATCHES(8);
-               }
+
+       if (priority > max_priority) { 
+            PrintDebug(VM_NONE, VCORE_NONE, "New highest priority match, reseting list\n");
+           max_priority = priority;
+
+           struct vnet_route_info *my_route, *tmp_route;
+
+           list_for_each_entry_safe(my_route, tmp_route, &match_list,match_node) {
+               list_del(&(my_route->match_node));
            }
-       }
+
+           list_add(&(route->match_node), &match_list);        
+           num_matches = 1;                                    
            
-       if (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0) {
-           if (route_def->dst_mac_qual != MAC_NOT) {
-               if (route_def->src_mac_qual == MAC_ANY) {
-                   UPDATE_MATCHES(6);
-               } else if ((route_def->src_mac_qual != MAC_NOT) && 
-                          (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {
-                   UPDATE_MATCHES(8);
-               }
-           }
-       }
+       } else if (priority == max_priority) {                      
+            PrintDebug(VM_NONE, VCORE_NONE, "Equal priority match, adding to list\n");
            
-       if ((route_def->dst_mac_qual == MAC_NOT) &&
-           (memcmp(route_def->dst_mac, hdr->dst_mac, 6) != 0)) {
-           if (route_def->src_mac_qual == MAC_ANY) {
-               UPDATE_MATCHES(5);
-           } else if ((route_def->src_mac_qual != MAC_NOT) && 
-                      (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0)) {     
-               UPDATE_MATCHES(7);
-           }
-       }
-       
-       if ((route_def->src_mac_qual == MAC_NOT) &&
-           (memcmp(route_def->src_mac, hdr->src_mac, 6) != 0)) {
-           if (route_def->dst_mac_qual == MAC_ANY) {
-               UPDATE_MATCHES(5);
-           } else if ((route_def->dst_mac_qual != MAC_NOT) &&
-                      (memcmp(route_def->dst_mac, hdr->dst_mac, 6) == 0)) {
-               UPDATE_MATCHES(7);
-           }
-       }
+           list_add(&(route->match_node), &match_list);        
+           num_matches++;                                      
+       }                                                       
        
-       // Default route
-       if ( (memcmp(route_def->src_mac, hdr->src_mac, 6) == 0) &&
-            (route_def->dst_mac_qual == MAC_NONE)) {
-           UPDATE_MATCHES(4);
-       }
     }
 
-    PrintDebug("VNET/P Core: match_route: Matches=%d\n", num_matches);
+    PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: match_route: Matches=%d\n", num_matches);
 
     if (num_matches <= 0) {
        return NULL;
     }
-
+    
     matches = (struct route_list *)Vnet_Malloc(sizeof(struct route_list) + 
                                               (sizeof(struct vnet_route_info *) * num_matches));
 
 
     if (!matches) {
-       PrintError("VNET/P Core: Unable to allocate matches\n");
+       PrintError(VM_NONE, VCORE_NONE, "VNET/P Core: Unable to allocate matches\n");
        return NULL;
     }
 
     matches->num_routes = num_matches;
 
-    {
-       int i = 0;
-       list_for_each_entry(route, &match_list, match_node) {
+    i=0;
+    list_for_each_entry(route, &match_list, match_node) {
+       if (i==num_matches) { 
+           // the list should never have more than num_matches on it...
+           PrintError(VM_NONE, VCORE_NONE, "Weird list behavior\n");
+           break;
+       } else {
            matches->routes[i++] = route;
        }
+       
     }
 
     return matches;
 }
 
-int v3_vnet_query_header(uint8_t src_mac[6], 
-                        uint8_t dest_mac[6],
+int v3_vnet_query_header(uint8_t src_mac[ETH_ALEN], 
+                        uint8_t dest_mac[ETH_ALEN],
                         int     recv,         // 0 = send, 1=recv
                         struct v3_vnet_header *header)
 {
     struct route_list *routes;
     struct vnet_route_info *r;
     struct v3_vnet_pkt p;
+    void *flags;
 
     p.size=14;
     p.data=p.header;
-    memcpy(p.header,dest_mac,6);
-    memcpy(p.header+6,src_mac,6);
+    memcpy(p.header,dest_mac,ETH_ALEN);
+    memcpy(p.header+ETH_ALEN,src_mac,ETH_ALEN);
     memset(p.header+12,0,2);
 
     p.src_type = LINK_EDGE;
     p.src_id = 0;
 
-    memcpy(header->src_mac,src_mac,6);
-    memcpy(header->dst_mac,dest_mac,6);
+    memcpy(header->src_mac,src_mac,ETH_ALEN);
+    memcpy(header->dst_mac,dest_mac,ETH_ALEN);
 
+
+    flags = vnet_lock_irqsave(vnet_state.lock);
     
     look_into_cache(&p,&routes);
 
     if (!routes) { 
        routes = match_route(&p);
        if (!routes) { 
-           PrintError("Cannot match route\n");
+           vnet_unlock_irqrestore(vnet_state.lock,flags);
+           PrintError(VM_NONE, VCORE_NONE, "Cannot match route\n");
            header->header_type=VNET_HEADER_NOMATCH;
            header->header_len=0;
            return -1;
@@ -521,16 +643,18 @@ int v3_vnet_query_header(uint8_t src_mac[6],
            add_route_to_cache(&p,routes);
        }
     }
+
+    vnet_unlock_irqrestore(vnet_state.lock,flags);
     
     if (routes->num_routes<1) { 
-       PrintError("Less than one route\n");
+       PrintError(VM_NONE, VCORE_NONE, "Less than one route\n");
        header->header_type=VNET_HEADER_NOMATCH;
        header->header_len=0;
        return -1;
     }
 
     if (routes->num_routes>1) { 
-       PrintError("More than one route, building header for the first one only\n");
+       PrintError(VM_NONE, VCORE_NONE, "More than one route, building header for the first one only\n");
     }
 
     r=routes->routes[0];
@@ -566,7 +690,7 @@ int v3_vnet_query_header(uint8_t src_mac[6],
            break;
 
        default:
-           PrintError("Unknown destination type\n");
+           PrintError(VM_NONE, VCORE_NONE, "Unknown destination type\n");
            return -1;
            break;
 
@@ -600,14 +724,14 @@ int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
     look_into_cache(pkt, &matched_routes);
 
     if (matched_routes == NULL) {  
-       PrintDebug("VNET/P Core: sending pkt - matching route\n");
+       PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: sending pkt - matching route\n");
        
        matched_routes = match_route(pkt);
        
        if (matched_routes) {
            add_route_to_cache(pkt, matched_routes);
        } else {
-           PrintDebug("VNET/P Core: Could not find route for packet... discarding packet\n");
+           PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: Could not find route for packet... discarding packet\n");
            vnet_unlock_irqrestore(vnet_state.lock, flags);
            return 0; /* do we return -1 here?*/
        }
@@ -615,7 +739,7 @@ int v3_vnet_send_pkt(struct v3_vnet_pkt * pkt, void * private_data) {
 
     vnet_unlock_irqrestore(vnet_state.lock, flags);
 
-    PrintDebug("VNET/P Core: send pkt route matches %d\n", matched_routes->num_routes);
+    PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: send pkt route matches %d\n", matched_routes->num_routes);
 
     for (i = 0; i < matched_routes->num_routes; i++) {
        struct vnet_route_info * route = matched_routes->routes[i];
@@ -670,7 +794,7 @@ int v3_vnet_add_dev(struct v3_vm_info * vm, uint8_t * mac,
        return -1;
     }
    
-    memcpy(new_dev->mac_addr, mac, 6);
+    memcpy(new_dev->mac_addr, mac, ETH_ALEN);
     new_dev->dev_ops.input = ops->input;
     new_dev->dev_ops.poll = ops->poll;
     new_dev->private_data = priv_data;
@@ -679,6 +803,8 @@ int v3_vnet_add_dev(struct v3_vm_info * vm, uint8_t * mac,
     new_dev->quote = quote<VNET_MAX_QUOTE ? quote : VNET_MAX_QUOTE;
     new_dev->poll = poll_state;
 
+    stop_vnet_kick_threads();
+
     flags = vnet_lock_irqsave(vnet_state.lock);
 
     if (dev_by_mac(mac) == NULL) {
@@ -690,18 +816,20 @@ int v3_vnet_add_dev(struct v3_vm_info * vm, uint8_t * mac,
            v3_enqueue(vnet_state.poll_devs, (addr_t)new_dev);
        }
     } else {
-       PrintError("VNET/P: Device with the same MAC has already been added\n");
+        PrintError(VM_NONE, VCORE_NONE,"VNET/P: Device with the same MAC has already been added\n");
     }
 
     vnet_unlock_irqrestore(vnet_state.lock, flags);
 
+    start_vnet_kick_threads();
+
     /* if the device was found previosly the id should still be 0 */
     if (new_dev->dev_id == 0) {
        Vnet_Print(0, "VNET/P Core: Device Already exists\n");
        return -1;
     }
 
-    PrintDebug("VNET/P Core: Add Device: dev_id %d\n", new_dev->dev_id);
+    PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: Add Device: dev_id %d\n", new_dev->dev_id);
 
     return new_dev->dev_id;
 }
@@ -711,6 +839,8 @@ int v3_vnet_del_dev(int dev_id){
     struct vnet_dev * dev = NULL;
     vnet_intr_flags_t flags;
 
+    stop_vnet_kick_threads();
+
     flags = vnet_lock_irqsave(vnet_state.lock);
        
     dev = dev_by_id(dev_id);
@@ -722,9 +852,11 @@ int v3_vnet_del_dev(int dev_id){
        
     vnet_unlock_irqrestore(vnet_state.lock, flags);
 
+    start_vnet_kick_threads();
+
     Vnet_Free(dev);
 
-    PrintDebug("VNET/P Core: Removed Device: dev_id %d\n", dev_id);
+    PrintDebug(VM_NONE, VCORE_NONE, "VNET/P Core: Removed Device: dev_id %d\n", dev_id);
 
     return 0;
 }
@@ -774,14 +906,14 @@ int v3_vnet_add_bridge(struct v3_vm_info * vm,
     vnet_unlock_irqrestore(vnet_state.lock, flags);
 
     if (bridge_free == 0) {
-       PrintError("VNET/P Core: Bridge already set\n");
+       PrintError(VM_NONE, VCORE_NONE, "VNET/P Core: Bridge already set\n");
        return -1;
     }
 
     tmp_bridge = (struct vnet_brg_dev *)Vnet_Malloc(sizeof(struct vnet_brg_dev));
 
     if (tmp_bridge == NULL) {
-       PrintError("VNET/P Core: Unable to allocate new bridge\n");
+       PrintError(VM_NONE, VCORE_NONE, "VNET/P Core: Unable to allocate new bridge\n");
        vnet_state.bridge = NULL;
        return -1;
     }
@@ -828,6 +960,7 @@ static int vnet_tx_flush(void * args){
     struct vnet_dev * dev = NULL;
     int more;
     int rc;
+    uint64_t noprogress_count;
 
     Vnet_Print(0, "VNET/P Polling Thread Starting ....\n");
 
@@ -841,11 +974,12 @@ static int vnet_tx_flush(void * args){
     struct v3_queue * tq = v3_create_queue();
 
     if (!tq) { 
-       PrintError("VNET/P polling thread cannot allocate queue\n");
+       PrintError(VM_NONE, VCORE_NONE, "VNET/P polling thread cannot allocate queue\n");
        return -1;
     }
 
-
+    noprogress_count = 0;
+    
     while (!vnet_thread_should_stop()) {
 
        more=0; // will indicate if any device has more work for us to do
@@ -871,14 +1005,20 @@ static int vnet_tx_flush(void * args){
            v3_enqueue(vnet_state.poll_devs, (addr_t)dev); 
        }
 
-       // Yield regardless of whether we handled any devices - need
-       // to allow other threads to run
+
        if (more) { 
-           // we have more to do, so we want to get back asap
+           noprogress_count=0;
+       } else {
+           if ( ! ((noprogress_count+1) < noprogress_count)) {
+               noprogress_count++;
+           }
+       }
+
+       // adaptively yield 
+       if ((!VNET_ADAPTIVE_TX_KICK) || (noprogress_count < VNET_NOPROGRESS_LIMIT)) { 
            V3_Yield();
        } else {
-           // put ourselves briefly to sleep if we we don't have more
-           V3_Yield_Timed(VNET_YIELD_USEC);
+           V3_Sleep(VNET_YIELD_USEC);
        }
 
     }
@@ -890,7 +1030,30 @@ static int vnet_tx_flush(void * args){
     return 0;
 }
 
-int v3_init_vnet() {
+static int start_vnet_kick_threads()
+{
+    int i;
+
+    for (i=0; i<VNET_NUM_TX_KICK_THREADS;i++) { 
+       char name[32];
+       snprintf(name,32,"vnetd-%d",i);
+       vnet_state.pkt_flush_thread[i] = vnet_start_thread(vnet_tx_flush, NULL, name);
+    }
+    return 0;
+}
+
+static int stop_vnet_kick_threads()
+{
+    int i;
+    for (i=0; i<VNET_NUM_TX_KICK_THREADS;i++) { 
+       vnet_thread_stop(vnet_state.pkt_flush_thread[i]);
+    }
+    return 0;
+}
+
+
+int v3_init_vnet() 
+{
     memset(&vnet_state, 0, sizeof(vnet_state));
        
     INIT_LIST_HEAD(&(vnet_state.routes));
@@ -900,58 +1063,62 @@ int v3_init_vnet() {
     vnet_state.num_routes = 0;
 
     if (vnet_lock_init(&(vnet_state.lock)) == -1){
-        PrintError("VNET/P: Fails to initiate lock\n");
+        PrintError(VM_NONE, VCORE_NONE, "VNET/P: Fails to initiate lock\n");
     }
 
     vnet_state.route_cache = vnet_create_htable(0, &hash_fn, &hash_eq);
     if (vnet_state.route_cache == NULL) {
-        PrintError("VNET/P: Fails to initiate route cache\n");
+        PrintError(VM_NONE, VCORE_NONE, "VNET/P: Fails to initiate route cache\n");
         return -1;
     }
 
     vnet_state.poll_devs = v3_create_queue();
 
-    vnet_state.pkt_flush_thread = vnet_start_thread(vnet_tx_flush, NULL, "vnetd-1");
+    start_vnet_kick_threads();
 
-    PrintDebug("VNET/P is initiated\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "VNET/P is initiated (%d tx kick threads active)\n",VNET_NUM_TX_KICK_THREADS);
 
     return 0;
 }
 
 
-void v3_deinit_vnet() {
+void v3_deinit_vnet() 
+{
 
+    PrintDebug(VM_NONE, VCORE_NONE, "Stopping kick threads\n");
+    stop_vnet_kick_threads();
+
+
+    PrintDebug(VM_NONE, VCORE_NONE, "Deiniting poll devices\n");
     v3_deinit_queue(vnet_state.poll_devs);
     Vnet_Free(vnet_state.poll_devs);
 
-    PrintDebug("Stopping flush thread\n");
-    // This will pause until the flush thread is gone
-    vnet_thread_stop(vnet_state.pkt_flush_thread);
+
     // At this point there should be no lock-holder
 
     Vnet_Free(vnet_state.poll_devs);
 
 
-    PrintDebug("Deiniting Device List\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "Deiniting Device List\n");
     // close any devices we have open
     deinit_devices_list();  
     
-    PrintDebug("Deiniting Route List\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "Deiniting Route List\n");
     // remove any routes we have
     deinit_routes_list();
 
-    PrintDebug("Freeing hash table\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "Freeing hash table\n");
     // remove the hash table
     vnet_free_htable(vnet_state.route_cache, 1, 1);
 
     
-    PrintDebug("Removing Bridge\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "Removing Bridge\n");
     // remove bridge if it was added
     if (vnet_state.bridge) { 
        Vnet_Free(vnet_state.bridge);
     }
 
-    PrintDebug("Deleting lock\n");
+    PrintDebug(VM_NONE, VCORE_NONE, "Deleting lock\n");
     // eliminate the lock
     vnet_lock_deinit(&(vnet_state.lock));