1 /* Palacios memory manager
5 #include <asm/page_64_types.h>
6 #include <linux/kernel.h>
7 #include <linux/list.h>
8 #include <linux/slab.h>
10 //static struct list_head pools;
16 #include "palacios/vmm.h"
19 static struct buddy_memzone ** memzones = NULL;
20 static uintptr_t * seed_addrs = NULL;
23 // alignment is in bytes
24 uintptr_t alloc_palacios_pgs(u64 num_pages, u32 alignment, int node_id, int constraints) {
26 int any = node_id==-1; // can allocate on any
27 int buddy_constraints=0;
29 if (constraints && constraints!=V3_ALLOC_PAGES_CONSTRAINT_4GB) {
30 ERROR("Unknown constraint mask 0x%x\n",constraints);
34 if (constraints & V3_ALLOC_PAGES_CONSTRAINT_4GB) {
35 buddy_constraints |= LWK_BUDDY_CONSTRAINT_4GB;
40 int cpu_id = get_cpu();
43 node_id = numa_cpu_to_node(cpu_id); // try first preferentially for the calling pcore
45 } else if (numa_num_nodes() == 1) {
47 } else if (node_id >= numa_num_nodes()) {
48 ERROR("Requesting memory from an invalid NUMA node. (Node: %d) (%d nodes on system)\n",
49 node_id, numa_num_nodes());
53 addr = buddy_alloc(memzones[node_id], get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT, buddy_constraints);
57 // do a scan to see if we can satisfy request on any node
58 for (i=0; i< numa_num_nodes(); i++) {
60 addr = buddy_alloc(memzones[i], get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT, buddy_constraints);
69 //DEBUG("Returning from alloc addr=%p, vaddr=%p\n", (void *)addr, __va(addr));
75 void free_palacios_pgs(uintptr_t pg_addr, u64 num_pages) {
76 int node_id = numa_addr_to_node(pg_addr);
78 //DEBUG("Freeing Memory page %p\n", (void *)pg_addr);
79 if (buddy_free(memzones[node_id], pg_addr, get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT)) {
80 // it is possible that the allocation was actually on a different zone,
81 // so, just to be sure, we'll try to dellocate on each
82 for (node_id=0;node_id<numa_num_nodes();node_id++) {
83 if (!buddy_free(memzones[node_id], pg_addr, get_order(num_pages * PAGE_SIZE) + PAGE_SHIFT)) {
84 // successfully freed on different zone, which is also OK
88 if (node_id==numa_num_nodes()) {
89 ERROR("Unable to free pages -addr=%p, numpages=%llu on any node\n",(void*)pg_addr,num_pages);
97 unsigned long long pow2(int i)
99 unsigned long long x=1;
100 for (;i!=0;i--) { x*=2; }
104 static unsigned long long get_palacios_mem_block_size(void)
106 char *s = v3_lookup_option("mem_block_size");
109 return V3_CONFIG_MEM_BLOCK_SIZE;
111 unsigned long long temp;
113 if (strict_strtoull(s,0,&temp)) {
114 return V3_CONFIG_MEM_BLOCK_SIZE; // odd...
121 int add_palacios_memory(struct v3_mem_region *r) {
126 struct v3_mem_region *keep;
128 INFO("Palacios Memory Add Request: type=%d, node=%d, base_addr=0x%llx, num_pages=%llu\n",r->type,r->node,r->base_addr,r->num_pages);
130 // fixup request regardless of its type
131 if (r->num_pages*4096 < get_palacios_mem_block_size()) {
132 WARNING("Allocating a memory pool smaller than the Palacios block size - may not be useful\n");
135 if (pow2(get_order(r->num_pages*PAGE_SIZE)) != r->num_pages) {
136 WARNING("Allocating a memory pool that is not a power of two (is %llu) - it will be rounded down!\n", r->num_pages);
137 r->num_pages=pow2(get_order(r->num_pages*PAGE_SIZE));
138 WARNING("Rounded request is for %llu pages\n", r->num_pages);
142 if (!(keep=palacios_alloc(sizeof(struct v3_mem_region)))) {
143 ERROR("Error allocating space for tracking region\n");
148 if (r->type==REQUESTED || r->type==REQUESTED32) {
151 INFO("Attempting to allocate %llu pages of %s memory\n", r->num_pages,
152 r->type==REQUESTED ? "64 bit (unrestricted)" :
153 r->type==REQUESTED32 ? "32 bit (restricted)" : "unknown (assuming 64 bit unrestricted)");
155 pgs = alloc_pages_node(r->node,
156 r->type==REQUESTED ? GFP_KERNEL :
157 r->type==REQUESTED32 ? GFP_DMA32 : GFP_KERNEL,
158 get_order(r->num_pages*PAGE_SIZE));
160 ERROR("Unable to satisfy allocation request\n");
164 r->base_addr = page_to_pfn(pgs) << PAGE_SHIFT;
170 node_id = numa_addr_to_node(r->base_addr);
173 ERROR("Error locating node for addr %p\n", (void *)(r->base_addr));
177 if ((node_id != r->node) && (r->node!=-1)) {
178 INFO("Memory add request is for node %d, but memory is in node %d\n",r->node,node_id);
181 pool_order = get_order(r->num_pages * PAGE_SIZE) + PAGE_SHIFT;
183 if (buddy_add_pool(memzones[node_id], r->base_addr, pool_order, keep)) {
184 ERROR("ALERT ALERT ALERT Unable to add pool to buddy allocator...\n");
185 if (r->type==REQUESTED || r->type==REQUESTED32) {
186 free_pages((uintptr_t)__va(r->base_addr), get_order(r->num_pages*PAGE_SIZE));
197 int remove_palacios_memory(struct v3_mem_region *req) {
198 int node_id = numa_addr_to_node(req->base_addr);
199 struct v3_mem_region *r;
201 if (buddy_remove_pool(memzones[node_id], req->base_addr, 0, (void**)(&r))) { //unforced remove
202 ERROR("Cannot remove memory at base address 0x%p\n", (void*)(req->base_addr));
207 if (r->type==REQUESTED || r->type==REQUESTED32) {
208 free_pages((uintptr_t)__va(r->base_addr), get_order(r->num_pages*PAGE_SIZE));
210 // user space responsible for onlining
219 static int handle_free(void *meta)
221 struct v3_mem_region *r = (struct v3_mem_region *)meta;
224 if (r->type==REQUESTED || r->type==REQUESTED32) {
225 //INFO("Freeing %llu pages at %p\n",r->num_pages,(void*)(r->base_addr));
226 free_pages((uintptr_t)__va(r->base_addr), get_order(r->num_pages*PAGE_SIZE));
228 // user space responsible for onlining
239 int palacios_deinit_mm( void ) {
244 for (i = 0; i < numa_num_nodes(); i++) {
247 INFO("Deiniting memory zone %d\n",i);
248 buddy_deinit(memzones[i],handle_free);
251 // note that the memory is not onlined here - offlining and onlining
252 // is the resposibility of the caller
255 // free the seed regions
256 INFO("Freeing seed addrs %d\n",i);
257 free_pages((uintptr_t)__va(seed_addrs[i]), MAX_ORDER - 1);
261 palacios_free(memzones);
262 palacios_free(seed_addrs);
268 int palacios_init_mm( void ) {
269 int num_nodes = numa_num_nodes();
272 INFO("memory manager init: MAX_ORDER=%d (%llu bytes)\n",MAX_ORDER, PAGE_SIZE*pow2(MAX_ORDER));
274 memzones = palacios_alloc_extended(sizeof(struct buddy_memzone *) * num_nodes, GFP_KERNEL,-1);
277 ERROR("Cannot allocate space for memory zones\n");
278 palacios_deinit_mm();
282 memset(memzones, 0, sizeof(struct buddy_memzone *) * num_nodes);
284 seed_addrs = palacios_alloc_extended(sizeof(uintptr_t) * num_nodes, GFP_KERNEL,-1);
287 ERROR("Cannot allocate space for seed addrs\n");
288 palacios_deinit_mm();
292 memset(seed_addrs, 0, sizeof(uintptr_t) * num_nodes);
294 for (node_id = 0; node_id < num_nodes; node_id++) {
295 struct buddy_memzone * zone = NULL;
297 // Seed the allocator with a small set of pages to allow initialization to complete.
298 // For now we will just grab some random pages, but in the future we will need to grab NUMA specific regions
299 // See: alloc_pages_node()
305 // attempt to first allocate below 4 GB for compatibility with
306 // 32 bit shadow paging
307 pgs = alloc_pages_node(node_id, GFP_DMA32, MAX_ORDER - 1);
311 INFO("Could not allocate initial memory block for node %d below 4GB\n", node_id);
313 pgs = alloc_pages_node(node_id, GFP_KERNEL, MAX_ORDER - 1);
316 INFO("Could not allocate initial memory block for node %d beloew 4GB\n", node_id);
318 ERROR("Could not allocate initial memory block for node %d without restrictions\n", node_id);
320 palacios_deinit_mm();
324 actual_node=numa_addr_to_node((uintptr_t)(page_to_pfn(pgs) << PAGE_SHIFT));
325 if (actual_node != node_id) {
326 WARNING("Initial 64 bit allocation attempt for node %d resulted in allocation on node %d\n",node_id,actual_node);
331 actual_node=numa_addr_to_node((uintptr_t)(page_to_pfn(pgs) << PAGE_SHIFT));
332 if (actual_node != node_id) {
333 WARNING("Initial 32bit-limited allocation attempt for node %d resulted in allocation on node %d\n",node_id,actual_node);
337 seed_addrs[node_id] = page_to_pfn(pgs) << PAGE_SHIFT;
340 // Initialization is done using the compile-time memory block size since
341 // at this point, we do not yet know what the run-time size is
342 zone = buddy_init(get_order(V3_CONFIG_MEM_BLOCK_SIZE) + PAGE_SHIFT, PAGE_SHIFT, node_id);
345 ERROR("Could not initialization memory management for node %d\n", node_id);
346 palacios_deinit_mm();
350 printk("Zone initialized, Adding seed region (order=%d)\n",
351 (MAX_ORDER - 1) + PAGE_SHIFT);
353 if (buddy_add_pool(zone, seed_addrs[node_id], (MAX_ORDER - 1) + PAGE_SHIFT,0)) {
354 ERROR("Could not add pool to buddy allocator\n");
355 palacios_deinit_mm();
359 memzones[node_id] = zone;