From: Peter Dinda <pdinda@northwestern.edu>
Date: Fri, 4 Oct 2013 22:12:23 +0000 (-0500)
Subject: Handle oddball NUMA alloc/dealloc cases when kernel provides memory on undesired... 
X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=1a47a4cd734ad40c20c7585023c8a290508b80d2

Handle oddball NUMA alloc/dealloc cases when kernel provides memory on undesired node
---

diff --git a/linux_module/buddy.c b/linux_module/buddy.c
index a68d786..43247b6 100644
--- a/linux_module/buddy.c
+++ b/linux_module/buddy.c
@@ -387,7 +387,7 @@ buddy_alloc(struct buddy_memzone *zone, unsigned long order, int constraints)
 /**
  * Returns a block of memory to the buddy system memory allocator.
  */
-void
+int
 buddy_free(
 	//!    Buddy system memory allocator object.
 	struct buddy_memzone *	zone,
@@ -422,9 +422,9 @@ buddy_free(
     pool = find_mempool(zone, addr);
 
     if ((pool == NULL) || (order > pool->pool_order)) {
-      WARNING("Attempted to free an invalid page address (%p) - pool=%p order=%lu\n", (void *)addr,pool,order);
-      palacios_spinlock_unlock_irqrestore(&(zone->lock), flags);
-	return;
+	WARNING("Attempted to free an invalid page address (%p) - pool=%p order=%lu\n", (void *)addr,pool,order);
+	palacios_spinlock_unlock_irqrestore(&(zone->lock), flags);
+	return -1;
     }
 
 
@@ -434,7 +434,7 @@ buddy_free(
     if (is_available(pool, block)) {
 	ERROR("Error: Freeing an available block\n");
 	palacios_spinlock_unlock_irqrestore(&(zone->lock), flags);
-	return;
+	return -1;
     }
 
     pool->num_free_blocks += (1UL << (order - zone->min_order));
@@ -465,6 +465,8 @@ buddy_free(
     list_add(&(block->link), &(zone->avail[order]));
 
     palacios_spinlock_unlock_irqrestore(&(zone->lock), flags);
+
+    return 0;
 }
 
 
diff --git a/linux_module/buddy.h b/linux_module/buddy.h
index 1017a3a..e0ef86d 100644
--- a/linux_module/buddy.h
+++ b/linux_module/buddy.h
@@ -103,7 +103,7 @@ buddy_alloc(struct buddy_memzone * zone,
 
 
 /* Free a physical address */
-extern void
+extern int
 buddy_free(struct buddy_memzone * zone,
 	   uintptr_t  addr,
 	   unsigned long order);
diff --git a/linux_module/mm.c b/linux_module/mm.c
index f394e6d..1b70b74 100644
--- a/linux_module/mm.c
+++ b/linux_module/mm.c
@@ -76,7 +76,19 @@ void free_palacios_pgs(uintptr_t pg_addr, u64 num_pages) {
     int node_id = numa_addr_to_node(pg_addr);
 
     //DEBUG("Freeing Memory page %p\n", (void *)pg_addr);
-    buddy_free(memzones[node_id], pg_addr, get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT);
+    if (buddy_free(memzones[node_id], pg_addr, get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT)) {
+      // it is possible that the allocation was actually on a different zone,
+      // so, just to be sure, we'll try to dellocate on each
+      for (node_id=0;node_id<numa_num_nodes();node_id++) { 
+        if (!buddy_free(memzones[node_id], pg_addr, get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT)) {
+	  // successfully freed on different zone, which is also OK
+	  break;
+        }
+      }
+      if (node_id==numa_num_nodes()) { 
+	ERROR("Unable to free pages -addr=%p, numpages=%llu on any node\n",(void*)pg_addr,num_pages);
+      }
+    }
     
     return;
 }
@@ -148,7 +160,7 @@ int add_palacios_memory(struct v3_mem_region *r) {
 	    ERROR("Unable to satisfy allocation request\n");
 	    palacios_free(keep);
 	    return -1;
-	}
+	} 
 	r->base_addr = page_to_pfn(pgs) << PAGE_SHIFT;
     }
 	
@@ -288,11 +300,13 @@ int palacios_init_mm( void ) {
 
 	{
 	    struct page * pgs;
-
+	    int actual_node;
+	      
 	    // attempt to first allocate below 4 GB for compatibility with
 	    // 32 bit shadow paging
 	    pgs = alloc_pages_node(node_id, GFP_DMA32, MAX_ORDER - 1);
 
+
 	    if (!pgs) {
 		INFO("Could not allocate initial memory block for node %d below 4GB\n", node_id);
 		
@@ -306,7 +320,18 @@ int palacios_init_mm( void ) {
 			palacios_deinit_mm();
 			return -1;
 		    }
+		} else {
+		  actual_node=numa_addr_to_node((uintptr_t)(page_to_pfn(pgs) << PAGE_SHIFT));
+		  if (actual_node != node_id) { 
+		    WARNING("Initial 64 bit allocation attempt for node %d resulted in allocation on node %d\n",node_id,actual_node);
+		  }
 		}
+		  
+	    } else {
+	      actual_node=numa_addr_to_node((uintptr_t)(page_to_pfn(pgs) << PAGE_SHIFT));
+	      if (actual_node != node_id) { 
+		WARNING("Initial 32bit-limited allocation attempt for node %d resulted in allocation on node %d\n",node_id,actual_node);
+	      }
 	    }
 
 	    seed_addrs[node_id] = page_to_pfn(pgs) << PAGE_SHIFT;