From: Peter Dinda Date: Wed, 1 Jun 2016 01:29:38 +0000 (-0500) Subject: Revised Nautilus Aerokernel Host Functionality X-Git-Url: http://v3vee.org/palacios/gitweb/gitweb.cgi?p=palacios.git;a=commitdiff_plain;h=2a9cf19fe40ce33f1c2728a41b18011dda365c06 Revised Nautilus Aerokernel Host Functionality - use of current allocator - use of enhanced virtual console - support for multiple VMs --- diff --git a/nautilus/console.c b/nautilus/console.c index f571a83..5a1afe2 100644 --- a/nautilus/console.c +++ b/nautilus/console.c @@ -3,8 +3,7 @@ */ #include -#include -#include +#include #include @@ -15,96 +14,129 @@ #include "palacios.h" -/* - This is a gruesome hack to allow the VM designated by the - host as "the_vm" to do I/O to the standard VGA text mode console -*/ +static void kbd_callback(nk_scancode_t scancode, void *priv) +{ + struct nk_vm_state *n = (struct nk_vm_state *) priv; + + struct v3_keyboard_event event = {0,scancode}; + + if (n && n->vm) { + v3_deliver_keyboard_event(n->vm, &event); + } else { + ERROR("Missing target for event... n=%p, n->vm=%p\n", n, n?n->vm:0); + } +} -extern void *the_vm; static void * palacios_tty_open(void * private_data, unsigned int width, unsigned int height) { - if (width!=80 || height!=25) { - ERROR("Console is wrong size\n"); - return 0; - } - INFO("Console connected\n"); - return (void*)1; + struct nk_vm_state *n = palacios_get_selected_vm(); + + if (!n) { + ERROR("Cannot create console without selected VM\n"); + return 0; + } + if (width!=80 || height!=25) { + ERROR("Console is wrong size\n"); + return 0; + } + + if (n->vc) { + ERROR("Cannot open multiple consoles per selected VM\n"); + return 0; + } + + + n->vc = nk_create_vc(n->name, + RAW_NOQUEUE, + 0x5f, + kbd_callback, + n); + + if (!n->vc) { + ERROR("Failed to create vc\n"); + return 0; + } + + nk_vc_clear_specific(n->vc,0x5f); + + return n; + } static int palacios_tty_cursor_set(void * console, int x, int y) { - if (console) { - term_setpos(x,y); - return 0; - } else { - return -1; - } + struct nk_vm_state *n = (struct nk_vm_state *) console; + + if (n && n->vc) { + nk_vc_setpos_specific(n->vc,x,y); + return 0; + } else { + return -1; + } } static int palacios_tty_character_set(void * console, int x, int y, char c, unsigned char style) { - if (console) { - term_putc(c,style,x,y); - return 0; - } else { - return -1; - } + struct nk_vm_state *n = (struct nk_vm_state *) console; + + if (n && n->vc) { + nk_vc_display_char_specific(n->vc,c,style,x,y); + nk_vc_setattr_specific(n->vc,style); + return 0; + } else { + return -1; + } } static int palacios_tty_scroll(void * console, int lines) { - if (console) { - int i; - for (i=0;ivc) { + int i; + for (i=0;ivc); } + return 0; + } else { + return -1; + } } static int palacios_set_text_resolution(void * console, int cols, int rows) { - if (console) { - if (cols!=80 || rows!=25) { - ERROR("Cannot change resolution\n"); - return -1; - } - else return 0; + if (console) { + if (cols!=80 || rows!=25) { + ERROR("Cannot change resolution\n"); + return -1; } else { - return -1; + return 0; } + } else { + return -1; + } } static int palacios_tty_update(void * console) { - return 0; + // not used for VC + return 0; } static void palacios_tty_close(void * console) { - if (console) { - term_clear(); - term_print("Palacios Console Finished\n"); - } -} + struct nk_vm_state *n = (struct nk_vm_state *) console; -static void kbd_callback(uint8_t scancode, uint8_t status) -{ - struct v3_keyboard_event event = {status,scancode}; - - //INFO("kbd callback scancode=%x\n",scancode); - if (the_vm) { - //INFO("Deliver scancode 0x%x\n",scancode); - v3_deliver_keyboard_event(the_vm, &event); - } + if (n && n->vc) { + nk_destroy_vc(n->vc); + } } + static struct v3_console_hooks palacios_console_hooks = { .open = palacios_tty_open, .set_cursor = palacios_tty_cursor_set, @@ -119,19 +151,15 @@ static struct v3_console_hooks palacios_console_hooks = { int nautilus_console_init(void) { - term_clear(); - term_print("Palacios Console\n"); - - V3_Init_Console(&palacios_console_hooks); - - kbd_register_callback(kbd_callback); - - return 0; + INFO("Palacios Console\n"); + + V3_Init_Console(&palacios_console_hooks); + + return 0; } int nautilus_console_deinit(void) { - // nothing to do return 0; } diff --git a/nautilus/palacios-nautilus-mm-test.c b/nautilus/palacios-nautilus-mm-test.c deleted file mode 100644 index ad39a7a..0000000 --- a/nautilus/palacios-nautilus-mm-test.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Unit test for palacios-nautilus memory allocator - * Can be invoked in init_palacios_nautilus_mm - */ - -#include // for panic - -#include "palacios-nautilus-mm.h" -#include "palacios-nautilus-mm-test.h" -#include "palacios.h" - -void test_palacios_mm(unsigned num_pages_limit) -{ - uintptr_t some_ptr; - unsigned int i = 0; - unsigned alignment = 4096; // gonna keep this constant for now since palacios only uses 4k pages - num_pages_limit -= 10; - - /* Allocate a gigantic piece of memory at once */ - some_ptr = alloc_palacios_pgs(num_pages_limit, alignment, 0, 0, 0); - if(!some_ptr) { - printk("ERROR IN PALACIOS-MM TEST: returned bogus address when not supposed to\n"); - panic(); - } - free_palacios_pgs(some_ptr, num_pages_limit); - - /* check if free_palacios_pg worked */ - some_ptr = alloc_palacios_pgs(100, alignment, 0, 0, 0); - if(!some_ptr) { - printk("FREE_PALACIOS_PGS DIDN'T WORK\n"); - panic(); - } - - /* Allocate many small pieces of memory consecutively */ - for(i = 0; i < num_pages_limit/100; i++) { - free_palacios_pgs(some_ptr, 100); - some_ptr = alloc_palacios_pgs(100, alignment, 0, 0, 0); - if (!some_ptr) { - printk("ERROR IN PALACIOS-MM TEST: returned bogus address when not supposed to\n"); - panic(); - } - } - - free_palacios_pgs(some_ptr, 100); - - uintptr_t ptrs[num_pages_limit]; - - for(i = 0; i < num_pages_limit/100; i++) { - ptrs[i] = alloc_palacios_pgs(100, alignment, 0, 0, 0); - } - - // first free random pages and then try to allocate them again - free_palacios_pgs(ptrs[0], 100); - free_palacios_pgs(ptrs[3], 100); - free_palacios_pgs(ptrs[4], 100); - - ptrs[0] = alloc_palacios_pgs(100, alignment, 0, 0, 0); - ptrs[3] = alloc_palacios_pgs(100, alignment, 0, 0, 0); - ptrs[4] = alloc_palacios_pgs(100, alignment, 0, 0, 0); - - for(i = 0; i < num_pages_limit/100; i++) { - free_palacios_pgs(ptrs[i], 100); - } - - - // TODO: WRITE MORE TESTS - printk("ALL TESTS PASSED - FREED ALL MEMORY\n"); -} diff --git a/nautilus/palacios-nautilus-mm-test.h b/nautilus/palacios-nautilus-mm-test.h deleted file mode 100644 index 89ad4ad..0000000 --- a/nautilus/palacios-nautilus-mm-test.h +++ /dev/null @@ -1 +0,0 @@ -void test_palacios_mm(unsigned num_pages_limit); diff --git a/nautilus/palacios-nautilus-mm.c b/nautilus/palacios-nautilus-mm.c deleted file mode 100644 index 4754922..0000000 --- a/nautilus/palacios-nautilus-mm.c +++ /dev/null @@ -1,217 +0,0 @@ -#include "palacios-nautilus-mm.h" -#include "palacios-nautilus-mm-test.h" - -/* - - Page-granularity memory management - - This impedence-matches between Nautilus's singular allocator (malloc/free) - and page-level memory allocation needed in Palacios. It does so via - a last-fit-optimized bitmap allocator that operates over a large pool - allocated from Nautilus at startup. - - Note that this allocation currently ignores NUMA and other constraints - as well as general filter expressions. - -*/ - -static uint64_t get_order(uint64_t n) -{ - uint64_t top_bit_pos; - - top_bit_pos = 63 - __builtin_clz(n); - - return top_bit_pos + !!(n & ~(1< 0); - pool.bitmap = palacios_alloc(bitmap_size); - - if (!pool.bitmap) { - ERROR("Palacios MM: Failed to allocate bitmap\n"); - return -1; - } - // note that this may not be aligned - pool.alloc_base_addr = (u64) palacios_alloc(PAGE_SIZE * num_nk_pages); - - if (!pool.alloc_base_addr) { - ERROR("Palacios MM: FAILED TO ALLOCATE MEMORY\n"); - return -1; - } else { - INFO("Palacios MM: success, alloc_base_addr=%p\n",pool.alloc_base_addr); - } - - // Align our memory to a page boundary - pool.base_addr = (u64) (((uint64_t)pool.alloc_base_addr & (~0xfffULL)) + PALACIOS_PAGE_SIZE); - - INFO("Palacios MM: success, cleaned up base_addr=%p\n",pool.base_addr); - - // We have one fewer pages than requested due to the need to align - // the result of the malloc - pool.num_pages = num_palacios_pages - 1 ; - pool.num_free_pages = num_palacios_pages - 1; - pool.pool_start = 0; - - // do unit test if desired - //test_palacios_mm(num_palacios_pages); - - return 0; - -} - - -int deinit_palacios_nautilus_mm(void) { - // free pages from nk - free((void*)pool.alloc_base_addr); pool.alloc_base_addr = 0; - free((void*)pool.bitmap); pool.bitmap = 0; - - return 0; -} - -static uintptr_t alloc_contig_pgs(u64 num_pages, u32 alignment) -{ - - int step = 1; - int i = 0; - int j = 0; - - if (num_pages > pool.num_free_pages) { - ERROR("ERROR(PALACIOS MM) : NOT ENOUGH MEMORY\n"); - return 0; - } - - //INFO("Allocating %llu pages (align=%lu)\n", num_pages, (unsigned long)alignment); - - if (!pool.bitmap || !pool.base_addr) { - ERROR("ERROR: Attempting to allocate from uninitialized memory pool \n"); - return 0; - } - - if (alignment > 0) { - if (alignment != 4096) { - ERROR("ERROR: cannot handle alignment that is not 4KB\n"); - return 0; - } - step = alignment / 4096; - } - - // scan pages from last search forward - for (i = pool.pool_start; i < (pool.num_pages - num_pages + 1) ; ) { - - for (j = i; j < (i+num_pages); j++) { - if (get_page_bit(j)) { - break; - } - } - - if (j==(i+num_pages)) { - for (j = i; j<(i+num_pages); j++) { - set_page_bit(j); - } - - pool.pool_start = j % pool.num_pages; - - return (void*) (pool.base_addr + (i * 4096)); - - } else { - i = j+1; - } - } - - - // scan from front if we didn't find it - for (i = 0; i < (pool.num_pages - num_pages + 1) ; ) { - - for (j = i; j < (i+num_pages); j++) { - if (get_page_bit(j)) { - break; - } - } - - if (j==(i+num_pages)) { - for (j = i; j<(i+num_pages); j++) { - set_page_bit(j); - } - - pool.pool_start = j % pool.num_pages; - - return (void*)( pool.base_addr + (i * 4096)); - - } else { - i = j+1; - } - } - - - ERROR("Palacios MM: ERROR! Cannot allocate memory...\n"); - ERROR("Palacios MM: Pool has %d pages, trying to allocate %d pages\n", pool.num_pages, num_pages); - - return 0; -} - -uintptr_t alloc_palacios_pgs(u64 num_pages, u32 alignment, int node_id, int (*filter_func)(void *paddr, void *filter_state), void *filter_state) -{ - uintptr_t addr = 0; - addr = alloc_contig_pgs(num_pages, alignment); - return addr; -} - - -void free_palacios_pgs(uintptr_t pg_addr, u64 num_pages) -{ - int pg_idx = ((u64)pg_addr - pool.base_addr) / PALACIOS_PAGE_SIZE; - int i = 0; - for (i = pg_idx; i < pg_idx+num_pages; i++) { - clear_page_bit(i); - } -} - - -void free_palacios_pg(uintptr_t pg_addr) -{ - free_palacios_pgs(pg_addr, 1); -} diff --git a/nautilus/palacios-nautilus-mm.h b/nautilus/palacios-nautilus-mm.h deleted file mode 100644 index 8a35683..0000000 --- a/nautilus/palacios-nautilus-mm.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef _PALACIOS_MM_H -#define _PALACIOS_MM_H - -#include -#include -#include -#include - - - -#include "palacios.h" - -#define PALACIOS_PAGE_SIZE 4096 - -static inline int get_page_bit(int index); -static inline void set_page_bit(int index); -static uintptr_t alloc_contig_pgs(u64 num_pages, u32 alignment); - - -int init_palacios_nautilus_mm(uint64_t memsize); -int deinit_palacios_nautilus_mm(void); -uintptr_t alloc_palacios_pgs(u64 num_pages, u32 alignment, int node_id, int (*filter_func)(void *paddr, void *filter_state), void *filter_state); -void free_palacios_pgs(uintptr_t base_addr, u64 num_pages); -void free_palacios_pg(uintptr_t base_addr); - -#endif diff --git a/nautilus/palacios-stubs.c b/nautilus/palacios-stubs.c index 61bf38f..a1b8183 100644 --- a/nautilus/palacios-stubs.c +++ b/nautilus/palacios-stubs.c @@ -1,15 +1,15 @@ #include -#include #include +#include #include #include +#include #include #include #include #include "palacios.h" -#include "palacios-nautilus-mm.h" #include "console.h" @@ -20,12 +20,9 @@ - Nautilus currently has a grand-unified allocator designed to help support parallel run-time integration. All of alloc/valloc/page - allocation are built on top of that. See palacios-nautilus-mm.c - for how this works for page allocation. + allocation are built on top of that. - For page allocation, constraints, NUMA, and filter expressions are ignored. - - To make this work, you also need updates on the Nautilus side. - (these will eventually get into the Nautilus repo) - thread migration is not supported currently - hooking of host interrupts is not supported currently. - Palacios can sleep, yield, wakeup, etc, but be aware @@ -36,11 +33,26 @@ - Do Nautilus regular startup to bring all cores to idle - From a kernel thread, ideally the init thread on core 0, do palacios_vmm_init(memory_size_bytes,options) - - Create, launch, etc, VMs using the Palacios v3_* functions - (note that these are NOT wrapped here) - - Console assumes void *the_vm is defined in the host, and it - is whatever a v3_create_vm() returned. This is the VM that - has console access (keyboard and screen). + - You can now use the Palacios v3_* functions, which are + not wrapped here. + - You need to keep the Nautilus VM state in sync with + the Palacios VM state. The protocol for this is: + 1. before doing a VM creation, call + palacios_inform_new_vm_pre(name) + this will also select the new vm for + the creation and going forward + then, once v3_create is done, call + palacios_inform_new_vm_post(name, vm) + 2. during execution, whenever you want to + manage a different VM, call + palacios_inform_select_vm(vm) + or + palacios_inform_select_vm_by_name(name) + It is OK to to select repeatedly, etc. + 3. after doing a VM free, call + palacios_inform_free_vm(name) + or + palacios_inform_free_selected_vm() - After you are done, do a palacios_vmm_deinit(); */ @@ -52,9 +64,12 @@ #define ALLOC_PAD 0 #define MAX_THREAD_NAME 32 - int run_nk_thread = 0; +static struct nk_vm_state vms[NR_VMS]; + +static struct nk_vm_state *selected_vm; + static struct v3_vm_info * irq_to_guest_map[256]; static unsigned int cpu_khz=-1; @@ -79,7 +94,6 @@ static int init_print_buffers(void) memset(print_buffer,0,sizeof(char*)*NR_CPUS); - for (i=0;i=0) { - printk(KERN_INFO "palacios (pcore %u vm %s vcore %u): %s", - cpu, - "some_guest", - vcore, - buf); + INFO_PRINT("palacios (pcore %u vm %s vcore %u): %s", + cpu, + "some_guest", + vcore, + buf); } else { - printk(KERN_INFO "palacios (pcore %u vm %s): %s", - cpu, - "some_guest", - buf); + INFO_PRINT(KERN_INFO "palacios (pcore %u vm %s): %s", + cpu, + "some_guest", + buf); } } else { - printk(KERN_INFO "palacios (pcore %u): %s", - cpu, - buf); + INFO_PRINT(KERN_INFO "palacios (pcore %u): %s", + cpu, + buf); } return; @@ -155,18 +168,25 @@ void *palacios_allocate_pages(int num_pages, unsigned int alignment, int node_id return NULL; } - pg_addr = (void *)alloc_palacios_pgs(num_pages, alignment, node_id, filter_func, filter_state); + // malloc currently guarantees alignment to the size of + // the allocation + pg_addr = (void *)malloc(num_pages*4096); if (!pg_addr) { ERROR("ALERT ALERT Page allocation has FAILED Warning (%d pages, alignment %d, node %d, filter_func %p, filter_state %p)\n",num_pages, alignment, node_id, filter_func, filter_state); return NULL; } + + if ((uint64_t)pg_addr & 0xfff) { + ERROR("ALERT ALERT Page allocation has surprise offset\n"); + return NULL; + } #if ALLOC_ZERO_MEM memset(pg_addr,0,num_pages*4096); #endif - // INFO("allocpages: %p (%llu pages) alignment=%u\n", pg_addr, num_pages, alignment); + //INFO("allocpages: %p (%llu pages) alignment=%u\n", pg_addr, num_pages, alignment); return pg_addr; } @@ -183,9 +203,9 @@ void palacios_free_pages(void * page_paddr, int num_pages) { ERROR("Ignoring free pages: 0x%p (0x%lx)for %d pages\n", page_paddr, (uintptr_t)page_paddr, num_pages); return; } - free_palacios_pgs((uintptr_t)page_paddr, num_pages); + free(page_paddr); - // INFO("freepages: %p (%llu pages) alignment=%u\n", page_paddr, num_pages); + INFO("freepages: %p (%llu pages) alignment=%u\n", page_paddr, num_pages); } @@ -710,9 +730,7 @@ static struct v3_os_hooks palacios_os_hooks = { }; - - -int palacios_vmm_init(uint64_t memsize, char * options) +int palacios_vmm_init(char * options) { int num_cpus = nautilus_info.sys.num_cpus; char * cpu_mask = NULL; @@ -740,15 +758,11 @@ int palacios_vmm_init(uint64_t memsize, char * options) } } - INFO("calling palacios-mm init\n"); - if (init_palacios_nautilus_mm(memsize)) { - ERROR("Failted to initialize memory management\n"); - return -1; - } - INFO("palacios-mm init done\n"); - + memset(irq_to_guest_map, 0, sizeof(struct v3_vm_info *) * 256); + memset(vms,0,sizeof(vms)); + if (init_print_buffers()) { INFO("Cannot initialize print buffers\n"); palacios_free(cpu_mask); @@ -757,8 +771,6 @@ int palacios_vmm_init(uint64_t memsize, char * options) INFO("printbuffer init done\n"); - //palacios_print_scoped(0, 0, "Hi%llu\n", 134217728); - INFO("NR_CPU: %d\n", NR_CPUS); INFO("palacios_init starting - calling init_v3\n"); @@ -795,7 +807,105 @@ int palacios_vmm_exit( void ) deinit_print_buffers(); - deinit_palacios_nautilus_mm(); // free memory from the allocator - return 0; } + + +void palacios_inform_new_vm_pre(char *name) +{ + int i; + for (i=0;ivm = vm; + INFO("Registered VM %p with name %s, node=%p, selected VM=%p\n", + vm, n->name, n, selected_vm); + } else { + ERROR("Cannot find VM with name \"%s\"\n",name); + } +} + +void palacios_inform_free_vm(char *name) +{ + struct nk_vm_state *n = palacios_find_vm_by_name(name); + + if (n==selected_vm) { + selected_vm = 0; + } + + if (n) { + n->vm = 0; + n->vc = 0; + n->name[0] = 0; + } + +} + +void palacios_inform_free_selected_vm() +{ + struct nk_vm_state *n = selected_vm; + + selected_vm = 0; + + if (n) { + n->vm = 0; + n->vc = 0; + n->name[0] = 0; + } +} + + +struct nk_vm_state *palacios_find_vm_by_name(char *name) +{ + int i; + for (i=0;i