From: Peter Dinda <pdinda@northwestern.edu>
Date: Wed, 7 Aug 2013 21:32:32 +0000 (-0500)
Subject: Memory manager fixes and assorted other fixes
X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=e26f7efb461ea8fba7eb0fd14809618c22dd475b

Memory manager fixes and assorted other fixes

- MM allocates seeed pool from 32 bit memory to facilitate 32 bit shadow guests
- MM page allocation falls back to any node (-1) if request
  cannot be satisfied for the specific requested node
- MM fixes for in-kernel allocated regions
- init now falls back gracefully on failure
- early procdir creation to support MM and VNET
---

diff --git a/linux_module/buddy.c b/linux_module/buddy.c
index 548c7ff..b4f2b81 100644
--- a/linux_module/buddy.c
+++ b/linux_module/buddy.c
@@ -496,8 +496,6 @@ static struct file_operations zone_proc_ops = {
 };
 
 
-extern struct proc_dir_entry * palacios_proc_dir;
-
 void buddy_deinit(struct buddy_memzone * zone) {
     unsigned long flags;
 
@@ -514,7 +512,7 @@ void buddy_deinit(struct buddy_memzone * zone) {
 	memset(proc_file_name, 0, 128);
 	snprintf(proc_file_name, 128, "v3-mem%d", zone->node_id);
 
-	remove_proc_entry(proc_file_name, palacios_proc_dir);
+	remove_proc_entry(proc_file_name, palacios_get_procdir());
     }
 
 
@@ -602,14 +600,15 @@ buddy_init(
 	char proc_file_name[128];
 
 	memset(proc_file_name, 0, 128);
-	snprintf(proc_file_name, 128, "v3-mem%d", zone->node_id);
+	snprintf(proc_file_name, 128, "v3-mem%u", zone->node_id);
 
-	zone_entry = create_proc_entry(proc_file_name, 0444, palacios_proc_dir);
+	zone_entry = create_proc_entry(proc_file_name, 0444, palacios_get_procdir());
 	if (zone_entry) {
 	    zone_entry->proc_fops = &zone_proc_ops;
 	    zone_entry->data = zone;
+	    INFO("Successfully created /proc/v3vee/v3-mem%d\n", zone->node_id);
 	} else {
-	    ERROR("Error creating memory zone proc file\n");
+	    ERROR("Cannot create /proc/v3vee/v3-mem%d\n", zone->node_id);
 	}
 
     }
diff --git a/linux_module/main.c b/linux_module/main.c
index 46de142..0f0325a 100644
--- a/linux_module/main.c
+++ b/linux_module/main.c
@@ -57,7 +57,7 @@ int mod_frees = 0;
 static int v3_major_num = 0;
 
 static struct v3_guest * guest_map[MAX_VMS] = {[0 ... MAX_VMS - 1] = 0};
-struct proc_dir_entry * palacios_proc_dir = NULL;
+static struct proc_dir_entry * palacios_proc_dir = NULL;
 
 struct class * v3_class = NULL;
 static struct cdev ctrl_dev;
@@ -232,6 +232,7 @@ static struct file_operations v3_ctrl_fops = {
 
 struct proc_dir_entry *palacios_get_procdir(void) 
 {
+    INFO("Returning procdir=%p\n",palacios_proc_dir);
     return palacios_proc_dir;
 }
 
@@ -388,7 +389,17 @@ static int __init v3_init(void) {
     LOCKCHECK_INIT();
     MEMCHECK_INIT();
 
-    palacios_init_mm();
+    palacios_proc_dir = proc_mkdir("v3vee", NULL);
+    if (!palacios_proc_dir) {
+	ERROR("Could not create proc entry\n");
+	ret = -1;
+	goto failure1;
+    }
+
+    // this will populate the v3vee tree...
+    if (palacios_init_mm()) { 
+	goto failure2;
+    }
 
     if (allow_devmem) {
       palacios_allow_devmem();
@@ -397,7 +408,6 @@ static int __init v3_init(void) {
     // Initialize Palacios
     palacios_vmm_init(options);
 
-
     // initialize extensions
     init_lnx_extensions();
 
@@ -405,7 +415,8 @@ static int __init v3_init(void) {
     v3_class = class_create(THIS_MODULE, "vms");
     if (IS_ERR(v3_class)) {
 	ERROR("Failed to register V3 VM device class\n");
-	return PTR_ERR(v3_class);
+	ret =  PTR_ERR(v3_class);
+	goto failure3;
     }
 
     INFO("intializing V3 Control device\n");
@@ -414,7 +425,7 @@ static int __init v3_init(void) {
 
     if (ret < 0) {
 	ERROR("Error registering device region for V3 devices\n");
-	goto failure2;
+	goto failure4;
     }
 
     v3_major_num = MAJOR(dev);
@@ -432,33 +443,40 @@ static int __init v3_init(void) {
 
     if (ret != 0) {
 	ERROR("Error adding v3 control device\n");
-	goto failure1;
+	goto failure5;
     }
 
-    palacios_proc_dir = proc_mkdir("v3vee", NULL);
-    if (palacios_proc_dir) {
+    {
 	struct proc_dir_entry *entry;
 
-	entry = create_proc_read_entry("v3-guests", 0444, palacios_proc_dir, 
-				       read_guests, NULL);
+	INFO("palacios_proc_dir=%p before v3-guests\n",palacios_proc_dir);
+	entry = create_proc_read_entry("v3-guests", 0444, palacios_proc_dir, read_guests, NULL);
         if (entry) {
 	    INFO("/proc/v3vee/v3-guests successfully created\n");
 	} else {
 	    ERROR("Could not create proc entry\n");
-	    goto failure1;
+	    goto failure6;
 	}
-	
-    } else {
-	ERROR("Could not create proc entry\n");
-	goto failure1;
     }
 	
     return 0;
 
- failure1:
+ failure6:
+    remove_proc_entry("v3-guests", palacios_proc_dir);
+ failure5:
     unregister_chrdev_region(MKDEV(v3_major_num, 0), MAX_VMS + 1);
- failure2:
+ failure4:
     class_destroy(v3_class);
+ failure3:
+    if (allow_devmem) {
+      palacios_restore_devmem();
+    }
+    palacios_deinit_mm();
+ failure2:
+    remove_proc_entry("v3vee", NULL);
+ failure1:   
+    MEMCHECK_DEINIT();
+    LOCKCHECK_DEINIT();
 
     return ret;
 }
diff --git a/linux_module/mm.c b/linux_module/mm.c
index a855261..01d4b75 100644
--- a/linux_module/mm.c
+++ b/linux_module/mm.c
@@ -22,12 +22,14 @@ static uintptr_t * seed_addrs = NULL;
 // alignment is in bytes
 uintptr_t alloc_palacios_pgs(u64 num_pages, u32 alignment, int node_id) {
     uintptr_t addr = 0; 
+    int any = node_id==-1; // can allocate on any
 
     if (node_id == -1) {
 	int cpu_id = get_cpu();
 	put_cpu();
 	
-	node_id = numa_cpu_to_node(cpu_id);
+	node_id = numa_cpu_to_node(cpu_id); // try first preferentially for the calling pcore
+
     } else if (numa_num_nodes() == 1) {
 	node_id = 0;
     } else if (node_id >= numa_num_nodes()) {
@@ -38,6 +40,20 @@ uintptr_t alloc_palacios_pgs(u64 num_pages, u32 alignment, int node_id) {
 
     addr = buddy_alloc(memzones[node_id], get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT);
 
+    if (!addr && any) { 
+	int i;
+	// do a scan to see if we can satisfy request on any node
+	for (i=0; i< numa_num_nodes(); i++) { 
+	    if (i!=node_id) { 
+		addr = buddy_alloc(memzones[i], get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT);
+		if (addr) {
+		    break;
+		}
+	    }
+	}
+    }
+		
+
     //DEBUG("Returning from alloc addr=%p, vaddr=%p\n", (void *)addr, __va(addr));
     return addr;
 }
@@ -67,15 +83,17 @@ int add_palacios_memory(struct v3_mem_region *r) {
 
     struct v3_mem_region *keep;
 
+    INFO("Palacios Memory Add Request: type=%d, node=%d, base_addr=0x%llx, num_pages=%llu\n",r->type,r->node,r->base_addr,r->num_pages);
+
     // fixup request regardless of its type
     if (r->num_pages*4096 < V3_CONFIG_MEM_BLOCK_SIZE) { 
 	WARNING("Allocating a memory pool smaller than the Palacios block size - may not be useful\n");
     }
 
-    if (pow2(get_order(r->num_pages*PAGE_SIZE)) != r->num_pages*PAGE_SIZE) { 
-	WARNING("Allocating a memory pool that is not a power of two (is %llu) - it will be rounded down!\n", r->num_pages*PAGE_SIZE);
+    if (pow2(get_order(r->num_pages*PAGE_SIZE)) != r->num_pages) { 
+	WARNING("Allocating a memory pool that is not a power of two (is %llu) - it will be rounded down!\n", r->num_pages);
 	r->num_pages=pow2(get_order(r->num_pages*PAGE_SIZE));
-	WARNING("Rounded power Allocating a memory pool that is not a power of two (rounded to %llu)\n", r->num_pages*PAGE_SIZE);
+	WARNING("Rounded request is for %llu pages\n", r->num_pages);
     }
 
 
@@ -86,10 +104,16 @@ int add_palacios_memory(struct v3_mem_region *r) {
 
 
     if (r->type==REQUESTED || r->type==REQUESTED32) { 
-	struct page * pgs = alloc_pages_node(r->node, 
-					     r->type==REQUESTED ? GFP_KERNEL :
-					     r->type==REQUESTED32 ? GFP_DMA32 : GFP_KERNEL, 
-					     get_order(r->num_pages));
+	struct page *pgs;
+
+	INFO("Attempting to allocate %llu pages of %s memory\n", r->num_pages,
+	     r->type==REQUESTED ? "64 bit (unrestricted)" : 
+	     r->type==REQUESTED32 ? "32 bit (restricted)" : "unknown (assuming 64 bit unrestricted)");
+	     
+	pgs = alloc_pages_node(r->node, 
+			       r->type==REQUESTED ? GFP_KERNEL :
+			       r->type==REQUESTED32 ? GFP_DMA32 : GFP_KERNEL, 
+			       get_order(r->num_pages*PAGE_SIZE));
 	if (!pgs) { 
 	    ERROR("Unable to satisfy allocation request\n");
 	    palacios_free(keep);
@@ -177,6 +201,8 @@ int palacios_init_mm( void ) {
     int num_nodes = numa_num_nodes();
     int node_id = 0;
 
+    INFO("memory manager init: MAX_ORDER=%d (%llu bytes)\n",MAX_ORDER, PAGE_SIZE*pow2(MAX_ORDER));
+
     memzones = palacios_alloc_extended(sizeof(struct buddy_memzone *) * num_nodes, GFP_KERNEL);
 
     if (!memzones) { 
@@ -205,13 +231,26 @@ int palacios_init_mm( void ) {
 	// See: alloc_pages_node()
 
 	{
-	    struct page * pgs = alloc_pages_node(node_id, GFP_KERNEL, MAX_ORDER - 1);
+	    struct page * pgs;
+
+	    // attempt to first allocate below 4 GB for compatibility with
+	    // 32 bit shadow paging
+	    pgs = alloc_pages_node(node_id, GFP_DMA32, MAX_ORDER - 1);
 
 	    if (!pgs) {
-		ERROR("Could not allocate initial memory block for node %d\n", node_id);
-		BUG_ON(!pgs);
-		palacios_deinit_mm();
-		return -1;
+		INFO("Could not allocate initial memory block for node %d beloew 4GB\n", node_id);
+		
+		pgs = alloc_pages_node(node_id, GFP_KERNEL, MAX_ORDER - 1);
+
+		if (!pgs) {
+		    INFO("Could not allocate initial memory block for node %d beloew 4GB\n", node_id);
+		    if (!pgs) {
+			ERROR("Could not allocate initial memory block for node %d without restrictions\n", node_id);
+			BUG_ON(!pgs);
+			palacios_deinit_mm();
+			return -1;
+		    }
+		}
 	    }
 
 	    seed_addrs[node_id] = page_to_pfn(pgs) << PAGE_SHIFT;